diff options
579 files changed, 9569 insertions, 3802 deletions
diff --git a/annotations/pom.xml b/annotations/pom.xml index 8a002f7c0f7..df4f14f2a7d 100644 --- a/annotations/pom.xml +++ b/annotations/pom.xml @@ -38,12 +38,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> diff --git a/client/go/cmd/log.go b/client/go/cmd/log.go index 4577e890959..7d3f6e95cd8 100644 --- a/client/go/cmd/log.go +++ b/client/go/cmd/log.go @@ -32,6 +32,8 @@ var logCmd = &cobra.Command{ Long: `Show the Vespa log. The logs shown can be limited to a relative or fixed period. All timestamps are shown in UTC. + +Logs for the past hour are shown if no arguments are given. `, Example: `$ vespa log 1h $ vespa log --nldequote=false 10m @@ -68,11 +70,13 @@ $ vespa log --follow`, } func parsePeriod(args []string) (time.Time, time.Time, error) { - if len(args) == 1 { - if fromArg != "" || toArg != "" { - return time.Time{}, time.Time{}, fmt.Errorf("cannot combine --from/--to with relative value: %s", args[0]) + relativePeriod := fromArg == "" || toArg == "" + if relativePeriod { + period := "1h" + if len(args) > 0 { + period = args[0] } - d, err := time.ParseDuration(args[0]) + d, err := time.ParseDuration(period) if err != nil { return time.Time{}, time.Time{}, err } @@ -82,6 +86,8 @@ func parsePeriod(args []string) (time.Time, time.Time, error) { to := time.Now() from := to.Add(d) return from, to, nil + } else if len(args) > 0 { + return time.Time{}, time.Time{}, fmt.Errorf("cannot combine --from/--to with relative value: %s", args[0]) } from, err := time.Parse(time.RFC3339, fromArg) if err != nil { diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index ab638a1da7d..db86df88fc5 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -283,7 +283,7 @@ public class MasterElectionTest extends FleetControllerTest { waitForCompleteCycles(); timer.advanceTime(options.zooKeeperSessionTimeout); waitForZookeeperDisconnected(); - // Noone can be master if server is unavailable + // No one can be master if server is unavailable log.log(Level.INFO, "Checking master status"); for (int i=0; i<fleetControllers.size(); ++i) { assertFalse("Index " + i, fleetControllers.get(i).isMaster()); diff --git a/config-application-package/pom.xml b/config-application-package/pom.xml index 2e429ea0ed6..7c84edce0f3 100644 --- a/config-application-package/pom.xml +++ b/config-application-package/pom.xml @@ -127,10 +127,6 @@ <artifactId>maven-compiler-plugin</artifactId> <configuration> <skip>false</skip> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> </configuration> </plugin> <plugin> diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/AbstractApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/AbstractApplicationPackage.java new file mode 100644 index 00000000000..c616784c7be --- /dev/null +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/AbstractApplicationPackage.java @@ -0,0 +1,44 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.application; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.Xml; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import java.util.Map; +import java.util.HashMap; + +/** + * Common code for all implementations of ApplicationPackage + * + * @author arnej + */ +public abstract class AbstractApplicationPackage implements ApplicationPackage { + + @Override + public Map<String,String> legacyOverrides() { + Map<String, String> result = new HashMap<>(); + try { + Document services = Xml.getDocument(getServices()); + NodeList legacyNodes = services.getElementsByTagName("legacy"); + for (int i=0; i < legacyNodes.getLength(); i++) { + var flagNodes = legacyNodes.item(i).getChildNodes(); + for (int j = 0; j < flagNodes.getLength(); ++j) { + var flagNode = flagNodes.item(j); + if (flagNode.getNodeType() == Node.ELEMENT_NODE) { + String key = flagNode.getNodeName(); + String value = flagNode.getTextContent(); + result.put(key, value); + } + } + } + } catch (Exception e) { + // nothing: This method does not validate that services.xml exists, or that it is valid xml. + } + return result; + } + +} diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 1223f438029..4cde4e7afaa 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -18,6 +18,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.config.model.application.AbstractApplicationPackage; import com.yahoo.io.HexDump; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; @@ -72,7 +73,7 @@ import static com.yahoo.text.Lowercase.toLowerCase; * * @author Vegard Havdal */ -public class FilesApplicationPackage implements ApplicationPackage { +public class FilesApplicationPackage extends AbstractApplicationPackage { /** * The name of the subdirectory (below the original application package root) diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index ec68ed73864..ae6f9373e16 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -103,6 +103,16 @@ public class FilesApplicationPackageTest { } @Test + public void testLegacyOverrides() throws IOException { + File appDir = new File("src/test/resources/app-legacy-overrides"); + ApplicationPackage app = FilesApplicationPackage.fromFile(appDir); + var overrides = app.legacyOverrides(); + assertEquals(2, overrides.size()); + assertEquals("something here", overrides.get("foo-bar")); + assertEquals("false", overrides.get("v7-geo-positions")); + } + + @Test public void failOnEmptyServicesXml() throws IOException { File appDir = temporaryFolder.newFolder(); IOUtils.copyDirectory(new File("src/test/resources/multienvapp"), appDir); diff --git a/config-application-package/src/test/resources/app-legacy-overrides/hosts.xml b/config-application-package/src/test/resources/app-legacy-overrides/hosts.xml new file mode 100644 index 00000000000..64a07644038 --- /dev/null +++ b/config-application-package/src/test/resources/app-legacy-overrides/hosts.xml @@ -0,0 +1,10 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts xmlns:deploy="vespa" xmlns:preprocess="properties"> + <preprocess:properties> + <node1.hostname>foo.yahoo.com</node1.hostname> + <node1.hostname deploy:environment="dev">bar.yahoo.com</node1.hostname> + </preprocess:properties> + <host name="${node1.hostname}"> + <alias>node1</alias> + </host> +</hosts> diff --git a/config-application-package/src/test/resources/app-legacy-overrides/schemas/music.sd b/config-application-package/src/test/resources/app-legacy-overrides/schemas/music.sd new file mode 100644 index 00000000000..7da7c49c162 --- /dev/null +++ b/config-application-package/src/test/resources/app-legacy-overrides/schemas/music.sd @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search music { + document music { + field f type string { + indexing: index | summary + } + } +} diff --git a/config-application-package/src/test/resources/app-legacy-overrides/services.xml b/config-application-package/src/test/resources/app-legacy-overrides/services.xml new file mode 100644 index 00000000000..5f8201336ef --- /dev/null +++ b/config-application-package/src/test/resources/app-legacy-overrides/services.xml @@ -0,0 +1,16 @@ +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version='1.0'> + <legacy> + <foo-bar>something here</foo-bar> + <v7-geo-positions>false</v7-geo-positions> + </legacy> + <admin version='2.0'> + <adminserver hostalias='node0'/> + </admin> + <content version='1.0' id='foo'> + <redundancy>1</redundancy> + <documents> + <document type="music.sd" mode="index" /> + </documents> + </content> +</services> diff --git a/config-lib/pom.xml b/config-lib/pom.xml index 0a8e6f884e2..fa540b5b387 100644 --- a/config-lib/pom.xml +++ b/config-lib/pom.xml @@ -38,12 +38,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> </plugin> <plugin> <groupId>com.yahoo.vespa</groupId> diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 8ad9f66ee6a..cac9d21ee1f 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -127,6 +127,7 @@ "public void writeMetaData()", "public java.util.Optional getAllocatedHosts()", "public java.util.Map getFileRegistries()", + "public java.util.Map legacyOverrides()", "public java.util.Collection getSearchDefinitions()", "public abstract java.util.Collection getSchemas()", "public com.yahoo.config.application.api.ApplicationPackage preprocess(com.yahoo.config.provision.Zone, com.yahoo.config.application.api.DeployLogger)" @@ -193,7 +194,7 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.config.provision.InstanceName, java.util.List, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout, java.util.List, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List)", + "public void <init>(com.yahoo.config.provision.InstanceName, java.util.List, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout, java.util.List, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List, java.time.Instant)", "public com.yahoo.config.provision.InstanceName name()", "public com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy upgradePolicy()", "public com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout upgradeRollout()", @@ -550,6 +551,19 @@ ], "fields": [] }, + "com.yahoo.config.application.api.TimeWindow$LocalDateRange": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public java.util.Optional start()", + "public java.util.Optional end()", + "public java.lang.String toString()" + ], + "fields": [] + }, "com.yahoo.config.application.api.TimeWindow": { "superClass": "java.lang.Object", "interfaces": [], @@ -560,9 +574,10 @@ "public java.util.List days()", "public java.util.List hours()", "public java.time.ZoneId zone()", + "public com.yahoo.config.application.api.TimeWindow$LocalDateRange dateRange()", "public boolean includes(java.time.Instant)", "public java.lang.String toString()", - "public static com.yahoo.config.application.api.TimeWindow from(java.lang.String, java.lang.String, java.lang.String)" + "public static com.yahoo.config.application.api.TimeWindow from(java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String)" ], "fields": [] }, diff --git a/config-model-api/pom.xml b/config-model-api/pom.xml index 49a9085134e..624c993d819 100644 --- a/config-model-api/pom.xml +++ b/config-model-api/pom.xml @@ -82,10 +82,6 @@ <artifactId>maven-compiler-plugin</artifactId> <configuration> <skip>false</skip> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> </configuration> </plugin> <plugin> diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 5c36de38c9b..d07df82fda1 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -238,6 +238,10 @@ public interface ApplicationPackage { return Collections.emptyMap(); } + default Map<String, String> legacyOverrides() { + return Collections.emptyMap(); + } + /** * @deprecated use {@link #getSchemas()} instead */ diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java index 23a5dce3c25..67ddb9ef83c 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java @@ -6,13 +6,16 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; +import java.time.Duration; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * The deployment spec for an application instance @@ -21,6 +24,9 @@ import java.util.stream.Collectors; */ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { + /** The maximum number of consecutive days Vespa upgrades are allowed to be blocked */ + private static final int maxUpgradeBlockingDays = 21; + /** The name of the instance this step deploys */ private final InstanceName name; @@ -40,7 +46,8 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { Optional<String> globalServiceId, Optional<AthenzService> athenzService, Notifications notifications, - List<Endpoint> endpoints) { + List<Endpoint> endpoints, + Instant now) { super(steps); this.name = name; this.upgradePolicy = upgradePolicy; @@ -52,6 +59,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { this.endpoints = List.copyOf(endpoints); validateZones(new HashSet<>(), new HashSet<>(), this); validateEndpoints(steps(), globalServiceId, this.endpoints); + validateChangeBlockers(changeBlockers, now); } public InstanceName name() { return name; } @@ -109,6 +117,36 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { } } + private void validateChangeBlockers(List<DeploymentSpec.ChangeBlocker> changeBlockers, Instant now) { + // Find all possible dates an upgrade block window can start + Stream<Instant> blockingFrom = changeBlockers.stream() + .filter(blocker -> blocker.blocksVersions()) + .map(blocker -> blocker.window()) + .map(window -> window.dateRange().start() + .map(date -> date.atStartOfDay(window.zone()) + .toInstant()) + .orElse(now)) + .distinct(); + if (!blockingFrom.allMatch(this::canUpgradeWithinDeadline)) { + throw new IllegalArgumentException("Cannot block Vespa upgrades for longer than " + + maxUpgradeBlockingDays + " consecutive days"); + } + } + + /** Returns whether this allows upgrade within deadline, relative to given instant */ + private boolean canUpgradeWithinDeadline(Instant instant) { + instant = instant.truncatedTo(ChronoUnit.HOURS); + Duration step = Duration.ofHours(1); + Duration max = Duration.ofDays(maxUpgradeBlockingDays); + for (Instant current = instant; !canUpgradeAt(current); current = current.plus(step)) { + Duration blocked = Duration.between(instant, current); + if (blocked.compareTo(max) > 0) { + return false; + } + } + return true; + } + /** Returns the upgrade policy of this, which is defaultPolicy if none is specified */ public DeploymentSpec.UpgradePolicy upgradePolicy() { return upgradePolicy; } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/TimeWindow.java b/config-model-api/src/main/java/com/yahoo/config/application/api/TimeWindow.java index 5a2b3a10fe1..746d7226d82 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/TimeWindow.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/TimeWindow.java @@ -4,20 +4,25 @@ package com.yahoo.config.application.api; import java.time.DateTimeException; import java.time.DayOfWeek; import java.time.Instant; +import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; +import java.time.format.DateTimeParseException; import java.time.temporal.ChronoField; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.EnumSet; import java.util.List; import java.util.Objects; +import java.util.Optional; +import java.util.Set; import java.util.function.BiFunction; import java.util.stream.Collectors; import java.util.stream.IntStream; /** - * This class represents a window of time for selected hours on selected days. + * This class represents a window of time for selected hours, days and dates. * * @author mpolden */ @@ -26,11 +31,20 @@ public class TimeWindow { private final List<DayOfWeek> days; private final List<Integer> hours; private final ZoneId zone; + private final LocalDateRange dateRange; - private TimeWindow(List<DayOfWeek> days, List<Integer> hours, ZoneId zone) { + private TimeWindow(List<DayOfWeek> days, List<Integer> hours, ZoneId zone, LocalDateRange dateRange) { this.days = Objects.requireNonNull(days).stream().distinct().sorted().collect(Collectors.toUnmodifiableList()); this.hours = Objects.requireNonNull(hours).stream().distinct().sorted().collect(Collectors.toUnmodifiableList()); this.zone = Objects.requireNonNull(zone); + this.dateRange = Objects.requireNonNull(dateRange); + if (days.isEmpty()) throw new IllegalArgumentException("At least one day must be specified"); + if (hours.isEmpty()) throw new IllegalArgumentException("At least one hour must be specified"); + for (var day : days) { + if (!dateRange.days().contains(day)) { + throw new IllegalArgumentException("Invalid day: " + dateRange + " does not contain " + day); + } + } } /** Returns days in this time window */ @@ -46,10 +60,17 @@ public class TimeWindow { /** Returns the time zone of this time window */ public ZoneId zone() { return zone; } + /** Returns the date range of this time window applies to */ + public LocalDateRange dateRange() { + return dateRange; + } + /** Returns whether the given instant is in this time window */ public boolean includes(Instant instant) { LocalDateTime dt = LocalDateTime.ofInstant(instant, zone); - return days.contains(dt.getDayOfWeek()) && hours.contains(dt.getHour()); + return days.contains(dt.getDayOfWeek()) && + hours.contains(dt.getHour()) && + dateRange.includes(dt.toLocalDate()); } @Override @@ -59,15 +80,20 @@ public class TimeWindow { " on " + days.stream().map(DayOfWeek::name) .map(String::toLowerCase) .collect(Collectors.toList()) + - " in " + zone; + " in time zone " + zone + " and " + dateRange.toString(); } /** Parse a time window from the given day, hour and time zone specification */ - public static TimeWindow from(String daySpec, String hourSpec, String zoneSpec) { - List<DayOfWeek> days = parse(daySpec, TimeWindow::parseDays); - List<Integer> hours = parse(hourSpec, TimeWindow::parseHours); - ZoneId zone = zoneFrom(zoneSpec); - return new TimeWindow(days, hours, zone); + public static TimeWindow from(String daySpec, String hourSpec, String zoneSpec, String dateStart, String dateEnd) { + LocalDateRange dateRange = LocalDateRange.from(dateStart, dateEnd); + List<DayOfWeek> days = daySpec.isEmpty() + ? List.copyOf(dateRange.days()) // Default to the days contained in the date range + : parse(daySpec, TimeWindow::parseDays); + List<Integer> hours = hourSpec.isEmpty() + ? IntStream.rangeClosed(0, 23).boxed().collect(Collectors.toList()) // All hours by default + : parse(hourSpec, TimeWindow::parseHours); + ZoneId zone = zoneFrom(zoneSpec.isEmpty() ? "UTC" : zoneSpec); + return new TimeWindow(days, hours, zone, dateRange); } /** Parse a specification, e.g. "1,4-5", using the given value parser */ @@ -97,7 +123,7 @@ public class TimeWindow { endInclusive)); } return IntStream.rangeClosed(start, end).boxed() - .collect(Collectors.toList()); + .collect(Collectors.toList()); } /** Returns a list of all days occurring between startInclusive and endInclusive */ @@ -109,16 +135,16 @@ public class TimeWindow { endInclusive)); } return IntStream.rangeClosed(start.getValue(), end.getValue()).boxed() - .map(DayOfWeek::of) - .collect(Collectors.toList()); + .map(DayOfWeek::of) + .collect(Collectors.toList()); } /** Parse day of week from string */ private static DayOfWeek dayFrom(String day) { return Arrays.stream(DayOfWeek.values()) - .filter(dayOfWeek -> day.length() >= 3 && dayOfWeek.name().toLowerCase().startsWith(day)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Invalid day '" + day + "'")); + .filter(dayOfWeek -> day.length() >= 3 && dayOfWeek.name().toLowerCase().startsWith(day)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Invalid day '" + day + "'")); } /** Parse hour from string */ @@ -139,4 +165,66 @@ public class TimeWindow { } } + /** A range of local dates, which may be unbounded */ + public static class LocalDateRange { + + private final Optional<LocalDate> start; + private final Optional<LocalDate> end; + + private LocalDateRange(Optional<LocalDate> start, Optional<LocalDate> end) { + this.start = Objects.requireNonNull(start); + this.end = Objects.requireNonNull(end); + if (start.isPresent() && end.isPresent() && start.get().isAfter(end.get())) { + throw new IllegalArgumentException("Invalid date range: start date " + start.get() + + " is after end date " + end.get()); + } + } + + /** Returns the starting date of this (inclusive), if any */ + public Optional<LocalDate> start() { + return start; + } + + /** Returns the ending date of this (inclusive), if any */ + public Optional<LocalDate> end() { + return end; + } + + /** Return days of week found in this range */ + private Set<DayOfWeek> days() { + if (start.isEmpty() || end.isEmpty()) return EnumSet.allOf(DayOfWeek.class); + Set<DayOfWeek> days = EnumSet.noneOf(DayOfWeek.class); + for (LocalDate date = start.get(); !date.isAfter(end.get()) && days.size() < 7; date = date.plusDays(1)) { + days.add(date.getDayOfWeek()); + } + return days; + } + + /** Returns whether includes the given date */ + private boolean includes(LocalDate date) { + if (start.isPresent() && date.isBefore(start.get())) return false; + if (end.isPresent() && date.isAfter(end.get())) return false; + return true; + } + + @Override + public String toString() { + return "date range [" + start.map(LocalDate::toString).orElse("any date") + + ", " + end.map(LocalDate::toString).orElse("any date") + "]"; + } + + private static LocalDateRange from(String start, String end) { + try { + return new LocalDateRange(optionalDate(start), optionalDate(end)); + } catch (DateTimeParseException e) { + throw new IllegalArgumentException("Could not parse date range '" + start + "' and '" + end + "'", e); + } + } + + private static Optional<LocalDate> optionalDate(String date) { + return Optional.of(date).filter(s -> !s.isEmpty()).map(LocalDate::parse); + } + + } + } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index aa985cd48bd..8f866654d56 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -27,7 +27,9 @@ import org.w3c.dom.Node; import java.io.IOException; import java.io.Reader; +import java.time.Clock; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -67,22 +69,28 @@ public class DeploymentSpecXmlReader { private static final String testerFlavorAttribute = "tester-flavor"; private final boolean validate; + private final Clock clock; private final List<DeprecatedElement> deprecatedElements = new ArrayList<>(); - /** Creates a validating reader */ + /** + * Create a deployment spec reader + * @param validate true to validate the input, false to accept any input which can be unambiguously parsed + * @param clock clock to use when validating time constraints + */ + public DeploymentSpecXmlReader(boolean validate, Clock clock) { + this.validate = validate; + this.clock = clock; + } + public DeploymentSpecXmlReader() { this(true); } - /** - * Creates a deployment spec reader - * - * @param validate true to validate the input, false to accept any input which can be unambiguously parsed - */ public DeploymentSpecXmlReader(boolean validate) { - this.validate = validate; + this(validate, Clock.systemUTC()); } + /** Reads a deployment spec from given reader */ public DeploymentSpec read(Reader reader) { try { return read(IOUtils.readAll(reader)); @@ -169,6 +177,7 @@ public class DeploymentSpecXmlReader { List<Endpoint> endpoints = readEndpoints(instanceTag, Optional.of(instanceNameString), steps); // Build and return instances with these values + Instant now = clock.instant(); return Arrays.stream(instanceNameString.split(",")) .map(name -> name.trim()) .map(name -> new DeploymentInstanceSpec(InstanceName.from(name), @@ -179,7 +188,8 @@ public class DeploymentSpecXmlReader { globalServiceId.asOptional(), athenzService, notifications, - endpoints)) + endpoints, + now)) .collect(Collectors.toList()); } @@ -361,7 +371,7 @@ public class DeploymentSpecXmlReader { */ private long longAttribute(String attributeName, Element tag) { String value = tag.getAttribute(attributeName); - if (value == null || value.isEmpty()) return 0; + if (value.isEmpty()) return 0; try { return Long.parseLong(value); } @@ -376,7 +386,7 @@ public class DeploymentSpecXmlReader { */ private Optional<Integer> optionalIntegerAttribute(String attributeName, Element tag) { String value = tag.getAttribute(attributeName); - if (value == null || value.isEmpty()) return Optional.empty(); + if (value.isEmpty()) return Optional.empty(); try { return Optional.of(Integer.parseInt(value)); } @@ -389,7 +399,7 @@ public class DeploymentSpecXmlReader { /** Returns the given non-blank attribute of tag as a string, if any */ private static Optional<String> stringAttribute(String attributeName, Element tag) { String value = tag.getAttribute(attributeName); - return Optional.ofNullable(value).filter(s -> !s.isBlank()); + return Optional.of(value).filter(s -> !s.isBlank()); } /** Returns the given non-blank attribute of tag or throw */ @@ -407,7 +417,7 @@ public class DeploymentSpecXmlReader { private Optional<String> readGlobalServiceId(Element environmentTag) { String globalServiceId = environmentTag.getAttribute("global-service-id"); - if (globalServiceId == null || globalServiceId.isEmpty()) return Optional.empty(); + if (globalServiceId.isEmpty()) return Optional.empty(); deprecate(environmentTag, List.of("global-service-id"), "See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax"); return Optional.of(globalServiceId); } @@ -430,9 +440,11 @@ public class DeploymentSpecXmlReader { String daySpec = tag.getAttribute("days"); String hourSpec = tag.getAttribute("hours"); String zoneSpec = tag.getAttribute("time-zone"); - if (zoneSpec.isEmpty()) zoneSpec = "UTC"; // default + String dateStart = tag.getAttribute("from-date"); + String dateEnd = tag.getAttribute("to-date"); + return new DeploymentSpec.ChangeBlocker(blockRevisions, blockVersions, - TimeWindow.from(daySpec, hourSpec, zoneSpec)); + TimeWindow.from(daySpec, hourSpec, zoneSpec, dateStart, dateEnd)); } /** Returns true if the given value is "true", or if it is missing */ diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index a0dba36fef5..6108c39f9d3 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -71,9 +71,9 @@ public interface ModelContext { interface FeatureFlags { @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Revisit in May or June 2021") default double defaultTermwiseLimit() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default boolean useThreePhaseUpdates() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { return "THROUGHPUT"; } @ModelFeatureFlag(owners = {"geirst, baldersheim"}) default int feedTaskLimit() { return 1000; } - @ModelFeatureFlag(owners = {"geirst, baldersheim"}) default int feedMasterTaskLimit() { return 0; } + @ModelFeatureFlag(owners = {"geirst, baldersheim"}) default int feedMasterTaskLimit() { return 1000; } @ModelFeatureFlag(owners = {"geirst, baldersheim"}) default String sharedFieldWriterExecutor() { return "NONE"; } @ModelFeatureFlag(owners = {"baldersheim"}) default String responseSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultNumResponseThreads() { return 2; } @@ -83,38 +83,36 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int metricsproxyNumThreads() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default int largeRankExpressionLimit() { return 8192; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int largeRankExpressionLimit() { return 8192; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxUnCommittedMemory() { return 130000; } - @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean ignoreMergeQueueLimit() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { return 16; } + @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { return 100; } + @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean ignoreMergeQueueLimit() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double containerShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default double diskBloatFactor() { throw new UnsupportedOperationException("TODO specify default value"); } - @ModelFeatureFlag(owners = {"baldersheim"}) default int docstoreCompressionLevel() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default double diskBloatFactor() { return 0.25; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default int docstoreCompressionLevel() { return 3; } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } @ModelFeatureFlag(owners = {"hmusum"}) default String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return ""; } - @ModelFeatureFlag(owners = {"arnej"}) default boolean requireConnectivityCheck() { return true; } @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitDisk() { return 0.8; } @ModelFeatureFlag(owners = {"hmusum"}) default double resourceLimitMemory() { return 0.8; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default double minNodeRatioPerGroup() { return 0.0; } - @ModelFeatureFlag(owners = {"arnej"}) default boolean newLocationBrokerLogic() { return true; } - @ModelFeatureFlag(owners = {"bjorncs"}, removeAfter = "7.504") default int maxConnectionLifeInHosted() { return 45; } - @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default int distributorMergeBusyWait() { return 10; } - @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean distributorEnhancedMaintenanceScheduling() { return false; } + @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default int distributorMergeBusyWait() { return 1; } + @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean distributorEnhancedMaintenanceScheduling() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean forwardIssuesAsErrors() { return true; } - @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default boolean asyncApplyBucketDiff() { return false; } + @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default boolean asyncApplyBucketDiff() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean ignoreThreadStackSizes() { return false; } - @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean unorderedMergeChaining() { return false; } + @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean unorderedMergeChaining() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } @ModelFeatureFlag(owners = {"arnej", "baldersheim"}) default boolean useV8DocManagerCfg() { return false; } @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } @ModelFeatureFlag(owners = {"hmusum"}) default boolean failDeploymentWithInvalidJvmOptions() { return false; } - @ModelFeatureFlag(owners = {"baldersheim"}) default double tlsSizeFraction() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "7.527") default double tlsSizeFraction() { return 0.02; } @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List<String> ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } + @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 43ccc34284f..2fa2ba83291 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -2,12 +2,15 @@ package com.yahoo.config.application.api; import com.google.common.collect.ImmutableSet; +import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; +import com.yahoo.test.ManualClock; import org.junit.Test; import java.io.StringReader; +import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.ZoneId; @@ -771,6 +774,7 @@ public class DeploymentSpecTest { " <instance id='default'>" + " <block-change revision='false' days='mon,tue' hours='15-16'/>" + " <block-change days='sat' hours='10' time-zone='CET'/>" + + " <block-change days='mon-sun' hours='0-23' time-zone='CET' from-date='2022-01-01' to-date='2022-01-15'/>" + " <prod>" + " <region active='true'>us-west-1</region>" + " </prod>" + @@ -778,7 +782,7 @@ public class DeploymentSpecTest { "</deployment>" ); DeploymentSpec spec = DeploymentSpec.fromXml(r); - assertEquals(2, spec.requireInstance("default").changeBlocker().size()); + assertEquals(3, spec.requireInstance("default").changeBlocker().size()); assertTrue(spec.requireInstance("default").changeBlocker().get(0).blocksVersions()); assertFalse(spec.requireInstance("default").changeBlocker().get(0).blocksRevisions()); assertEquals(ZoneId.of("UTC"), spec.requireInstance("default").changeBlocker().get(0).window().zone()); @@ -795,6 +799,8 @@ public class DeploymentSpecTest { assertTrue(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-23T09:15:30.00Z"))); assertFalse(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-23T08:15:30.00Z"))); // 10 in CET assertTrue(spec.requireInstance("default").canUpgradeAt(Instant.parse("2017-09-23T10:15:30.00Z"))); + + assertFalse(spec.requireInstance("default").canUpgradeAt(Instant.parse("2022-01-15T16:00:00.00Z"))); } @Test @@ -812,11 +818,13 @@ public class DeploymentSpecTest { DeploymentSpec spec = DeploymentSpec.fromXml(r); - String inheritedChangeBlocker = "change blocker revision=false version=true window=time window for hour(s) [15, 16] on [monday, tuesday] in UTC"; + String inheritedChangeBlocker = "change blocker revision=false version=true window=time window for hour(s) " + + "[15, 16] on [monday, tuesday] in time zone UTC and date range [any date, any date]"; assertEquals(2, spec.requireInstance("instance1").changeBlocker().size()); assertEquals(inheritedChangeBlocker, spec.requireInstance("instance1").changeBlocker().get(0).toString()); - assertEquals("change blocker revision=true version=true window=time window for hour(s) [10] on [saturday] in CET", + assertEquals("change blocker revision=true version=true window=time window for hour(s) [10] on " + + "[saturday] in time zone CET and date range [any date, any date]", spec.requireInstance("instance1").changeBlocker().get(1).toString()); assertEquals(1, spec.requireInstance("instance2").changeBlocker().size()); @@ -1269,10 +1277,45 @@ public class DeploymentSpecTest { spec.requireInstance("main").endpoints()); } + @Test + public void disallowExcessiveUpgradeBlocking() { + List<String> specs = List.of( + "<deployment>\n" + + " <block-change/>\n" + + "</deployment>", + + "<deployment>\n" + + " <block-change days=\"mon-wed\"/>\n" + + " <block-change days=\"tue-sun\"/>\n" + + "</deployment>", + + "<deployment>\n" + + " <block-change to-date=\"2023-01-01\"/>\n" + + "</deployment>", + + // Convoluted example of blocking too long + "<deployment>\n" + + " <block-change days=\"sat-sun\"/>\n" + + " <block-change days=\"mon-fri\" hours=\"0-10\" from-date=\"2023-01-01\" to-date=\"2023-01-15\"/>\n" + + " <block-change days=\"mon-fri\" hours=\"11-23\" from-date=\"2023-01-01\" to-date=\"2023-01-15\"/>\n" + + " <block-change from-date=\"2023-01-14\" to-date=\"2023-01-31\"/>" + + "</deployment>" + ); + ManualClock clock = new ManualClock(); + clock.setInstant(Instant.parse("2022-01-05T15:00:00.00Z")); + for (var spec : specs) { + assertInvalid(spec, "Cannot block Vespa upgrades for longer than 21 consecutive days", clock); + } + } + private static void assertInvalid(String deploymentSpec, String errorMessagePart) { + assertInvalid(deploymentSpec, errorMessagePart, new ManualClock()); + } + + private static void assertInvalid(String deploymentSpec, String errorMessagePart, Clock clock) { if (errorMessagePart.isEmpty()) throw new IllegalArgumentException("Message part must be non-empty"); try { - DeploymentSpec.fromXml(deploymentSpec); + new DeploymentSpecXmlReader(true, clock).read(deploymentSpec); fail("Expected exception"); } catch (IllegalArgumentException e) { assertTrue("\"" + e.getMessage() + "\" contains \"" + errorMessagePart + "\"", diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/TimeWindowTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/TimeWindowTest.java index 98a53dfd3df..2403066868b 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/TimeWindowTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/TimeWindowTest.java @@ -3,7 +3,11 @@ package com.yahoo.config.application.api; import org.junit.Test; +import java.time.DayOfWeek; import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static java.time.DayOfWeek.FRIDAY; import static java.time.DayOfWeek.MONDAY; @@ -11,7 +15,6 @@ import static java.time.DayOfWeek.SATURDAY; import static java.time.DayOfWeek.THURSDAY; import static java.time.DayOfWeek.TUESDAY; import static java.time.DayOfWeek.WEDNESDAY; -import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -25,71 +28,112 @@ public class TimeWindowTest { @Test public void includesInstant() { { - TimeWindow tw = TimeWindow.from("mon", "10,11", "UTC"); + TimeWindow tw = TimeWindow.from("mon", "10,11", "UTC", "", ""); Instant i0 = Instant.parse("2017-09-17T11:15:30.00Z"); // Wrong day Instant i1 = Instant.parse("2017-09-18T09:15:30.00Z"); // Wrong hour Instant i2 = Instant.parse("2017-09-18T10:15:30.00Z"); Instant i3 = Instant.parse("2017-09-18T11:15:30.00Z"); Instant i4 = Instant.parse("2017-09-18T12:15:30.00Z"); // Wrong hour Instant i5 = Instant.parse("2017-09-19T11:15:30.00Z"); // Wrong day - - assertFalse("Instant " + i0 + " is not in window", tw.includes(i0)); - assertFalse("Instant " + i1 + " is not in window", tw.includes(i1)); - assertTrue("Instant " + i2 + " is in window", tw.includes(i2)); - assertTrue("Instant " + i3 + " is in window", tw.includes(i3)); - assertFalse("Instant " + i4 + " is not in window", tw.includes(i4)); - assertFalse("Instant " + i5 + " is not in window", tw.includes(i5)); + assertOutside(tw, i0); + assertOutside(tw, i1); + assertInside(tw, i2); + assertInside(tw, i3); + assertOutside(tw, i4); + assertOutside(tw, i5); } { - TimeWindow tw = TimeWindow.from("mon", "12,13", "CET"); + TimeWindow tw = TimeWindow.from("mon", "12,13", "CET", "", ""); Instant i0 = Instant.parse("2017-09-17T11:15:30.00Z"); Instant i1 = Instant.parse("2017-09-18T09:15:30.00Z"); Instant i2 = Instant.parse("2017-09-18T10:15:30.00Z"); // Including offset this matches hour 12 Instant i3 = Instant.parse("2017-09-18T11:15:30.00Z"); // Including offset this matches hour 13 Instant i4 = Instant.parse("2017-09-18T12:15:30.00Z"); Instant i5 = Instant.parse("2017-09-19T11:15:30.00Z"); - assertFalse("Instant " + i0 + " is not in window", tw.includes(i0)); - assertFalse("Instant " + i1 + " is not in window", tw.includes(i1)); - assertTrue("Instant " + i2 + " is in window", tw.includes(i2)); - assertTrue("Instant " + i3 + " is in window", tw.includes(i3)); - assertFalse("Instant " + i4 + " is not in window", tw.includes(i4)); - assertFalse("Instant " + i5 + " is not in window", tw.includes(i5)); + assertOutside(tw, i0); + assertOutside(tw, i1); + assertInside(tw, i2); + assertInside(tw, i3); + assertOutside(tw, i4); + assertOutside(tw, i5); + } + { + TimeWindow tw = TimeWindow.from("mon-sun", "0-23", "CET", "2022-01-15", "2022-02-15"); + Instant i0 = Instant.parse("2022-01-14T12:00:00.00Z"); // Before window + Instant i1 = Instant.parse("2022-01-14T23:00:00.00Z"); // Inside window because of time zone offset + Instant i2 = Instant.parse("2022-02-05T12:00:00.00Z"); + Instant i3 = Instant.parse("2022-02-14T23:00:00.00Z"); + Instant i4 = Instant.parse("2022-02-15T23:00:00.00Z"); // After window because of time zone offset + Instant i5 = Instant.parse("2022-02-16T12:00:00.00Z"); // After window + assertOutside(tw, i0); + assertInside(tw, i1); + assertInside(tw, i2); + assertInside(tw, i3); + assertOutside(tw, i4); + assertOutside(tw, i5); + + TimeWindow tw2 = TimeWindow.from("sun", "1", "CET", "2022-01-01", "2022-01-02"); + Instant i6 = Instant.parse("2022-01-01T00:00:00.00Z"); // Wrong day + Instant i7 = Instant.parse("2022-01-02T01:00:00.00Z"); // Wrong hour because of time zone offset + Instant i8 = Instant.parse("2022-01-02T00:00:00.00Z"); + assertOutside(tw2, i6); + assertOutside(tw2, i7); + assertInside(tw2, i8); + + TimeWindow tw3 = TimeWindow.from("", "", "CET", "2022-01-02", ""); + Instant i9 = Instant.parse("2022-02-15T00:00:00.00Z"); + assertOutside(tw3, i6); + assertInside(tw3, i7); + assertInside(tw3, i8); + assertInside(tw3, i9); } } @Test public void validWindows() { { - TimeWindow fz = TimeWindow.from("fri", "8,17-19", "UTC"); - assertEquals(asList(FRIDAY), fz.days()); - assertEquals(asList(8, 17, 18, 19), fz.hours()); + TimeWindow tw = TimeWindow.from("fri", "8,17-19", "UTC", "", ""); + assertEquals(List.of(FRIDAY), tw.days()); + assertEquals(List.of(8, 17, 18, 19), tw.hours()); } { - TimeWindow fz = TimeWindow.from("sat,", "8,17-19", "UTC"); - assertEquals(asList(SATURDAY), fz.days()); - assertEquals(asList(8, 17, 18, 19), fz.hours()); + TimeWindow tw = TimeWindow.from("sat,", "8,17-19", "UTC", "", ""); + assertEquals(List.of(SATURDAY), tw.days()); + assertEquals(List.of(8, 17, 18, 19), tw.hours()); } { - TimeWindow fz = TimeWindow.from("tue,sat", "0,3,7,10", "UTC"); - assertEquals(asList(TUESDAY, SATURDAY), fz.days()); - assertEquals(asList(0, 3, 7, 10), fz.hours()); + TimeWindow tw = TimeWindow.from("tue,sat", "0,3,7,10", "UTC", "", ""); + assertEquals(List.of(TUESDAY, SATURDAY), tw.days()); + assertEquals(List.of(0, 3, 7, 10), tw.hours()); } { - TimeWindow fz = TimeWindow.from("mon,wed-thu", "0,17-19", "UTC"); - assertEquals(asList(MONDAY, WEDNESDAY, THURSDAY), fz.days()); - assertEquals(asList(0, 17, 18, 19), fz.hours()); + TimeWindow tw = TimeWindow.from("mon,wed-thu", "0,17-19", "UTC", "", ""); + assertEquals(List.of(MONDAY, WEDNESDAY, THURSDAY), tw.days()); + assertEquals(List.of(0, 17, 18, 19), tw.hours()); + } + { // Empty results in default values + TimeWindow tw = TimeWindow.from("", "", "", "", ""); + assertEquals(List.of(DayOfWeek.values()), tw.days()); + assertEquals(IntStream.rangeClosed(0, 23).boxed().collect(Collectors.toList()), tw.hours()); + assertEquals("UTC", tw.zone().getId()); } { // Full day names is allowed - TimeWindow fz = TimeWindow.from("monday,wednesday-thursday", "0,17-19", "UTC"); - assertEquals(asList(MONDAY, WEDNESDAY, THURSDAY), fz.days()); - assertEquals(asList(0, 17, 18, 19), fz.hours()); + TimeWindow tw = TimeWindow.from("monday,wednesday-thursday", "0,17-19", "UTC", "", ""); + assertEquals(List.of(MONDAY, WEDNESDAY, THURSDAY), tw.days()); + assertEquals(List.of(0, 17, 18, 19), tw.hours()); } { // Duplicate day and overlapping range is allowed - TimeWindow fz = TimeWindow.from("mon,wed-thu,mon", "3,1-4", "UTC"); - assertEquals(asList(MONDAY, WEDNESDAY, THURSDAY), fz.days()); - assertEquals(asList(1, 2, 3, 4), fz.hours()); + TimeWindow tw = TimeWindow.from("mon,wed-thu,mon", "3,1-4", "UTC", "", ""); + assertEquals(List.of(MONDAY, WEDNESDAY, THURSDAY), tw.days()); + assertEquals(List.of(1, 2, 3, 4), tw.hours()); + } + { // Default to days contained in the date range + TimeWindow tw = TimeWindow.from("", "", "", "2022-01-11", "2022-01-14"); + assertEquals(List.of(TUESDAY, WEDNESDAY, THURSDAY, FRIDAY), tw.days()); + TimeWindow tw2 = TimeWindow.from("", "", "", "2022-01-01", "2100-01-01"); + assertEquals(List.of(DayOfWeek.values()), tw2.days()); } } @@ -99,7 +143,6 @@ public class TimeWindowTest { assertInvalidZone("foo", "Invalid time zone 'foo'"); // Malformed day input - assertInvalidDays("", "Invalid day ''"); assertInvalidDays("foo-", "Invalid range 'foo-'"); assertInvalidDays("foo", "Invalid day 'foo'"); assertInvalidDays("f", "Invalid day 'f'"); @@ -107,16 +150,29 @@ public class TimeWindowTest { assertInvalidDays("fri-tue", "Invalid day range 'fri-tue'"); // Malformed hour input - assertInvalidHours("", "Invalid hour ''"); assertInvalidHours("24", "Invalid hour '24'"); assertInvalidHours("-1-9", "Invalid range '-1-9'"); // Window crossing day boundary is disallowed assertInvalidHours("23-1", "Invalid hour range '23-1'"); + + // Invalid date range + assertInvalidDateRange("", "foo", "bar", "Could not parse date range 'foo' and 'bar'"); + assertInvalidDateRange("", "2022-01-15", "2022-01-01", "Invalid date range: start date 2022-01-15 is after end date 2022-01-01"); + assertInvalidDateRange("wed", "2022-01-06", "2022-01-09", "Invalid day: date range [2022-01-06, 2022-01-09] does not contain WEDNESDAY"); + assertInvalidDateRange("mon-sun", "2022-01-03", "2022-01-07", "Invalid day: date range [2022-01-03, 2022-01-07] does not contain SATURDAY"); + } + + private static void assertOutside(TimeWindow window, Instant instant) { + assertFalse("Instant " + instant + " is not in window", window.includes(instant)); + } + + private static void assertInside(TimeWindow window, Instant instant) { + assertTrue("Instant " + instant + " is in window", window.includes(instant)); } private static void assertInvalidZone(String zoneSpec, String exceptionMessage) { try { - TimeWindow.from("mon", "1", zoneSpec); + TimeWindow.from("mon", "1", zoneSpec, "", ""); fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals(exceptionMessage, e.getMessage()); @@ -125,7 +181,7 @@ public class TimeWindowTest { private static void assertInvalidDays(String daySpec, String exceptionMessage) { try { - TimeWindow.from(daySpec, "1", "UTC"); + TimeWindow.from(daySpec, "1", "UTC", "", ""); fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals(exceptionMessage, e.getMessage()); @@ -134,11 +190,20 @@ public class TimeWindowTest { private static void assertInvalidHours(String hourSpec, String exceptionMessage) { try { - TimeWindow.from("mon", hourSpec, "UTC"); + TimeWindow.from("mon", hourSpec, "UTC", "", ""); fail("Expected exception"); } catch (IllegalArgumentException e) { assertEquals(exceptionMessage, e.getMessage()); } } + private static void assertInvalidDateRange(String daySpec, String startDate, String endDate, String message) { + try { + TimeWindow.from(daySpec, "", "UTC", startDate, endDate); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals(message, e.getMessage()); + } + } + } diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 79f106993df..430e471afd0 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -50,12 +50,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> </plugin> <plugin> <groupId>org.codehaus.mojo</groupId> diff --git a/config-model/pom.xml b/config-model/pom.xml index 18b8432645e..dc7bec27a3b 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -288,6 +288,22 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>biz.aQute.bnd</groupId> + <artifactId>biz.aQute.bndlib</artifactId> + <version>6.1.0</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + <exclusion> + <!-- These are not needed for our use of bndlib --> + <groupId>org.osgi</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> </dependencies> <build> @@ -306,8 +322,6 @@ <arg>-Xlint:-rawtypes</arg> <arg>-Xlint:-unchecked</arg> <arg>-Xlint:-serial</arg> - <arg>-Xlint:-cast</arg> - <arg>-Xlint:-overloads</arg> <arg>-Werror</arg> </compilerArgs> </configuration> diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 4591578d7e5..8d192414871 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -133,6 +133,10 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter for (Element servicesElement : children) { String tagName = servicesElement.getTagName(); + if (tagName.equals("legacy")) { + // for enabling legacy features from old vespa versions + continue; + } if (tagName.equals("config")) { // TODO: disallow on Vespa 8 continue; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index e645fec5520..c148bb0e6e4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -40,9 +40,9 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean useThreePhaseUpdates = false; private double defaultTermwiseLimit = 1.0; private String jvmGCOptions = null; - private String sequencerType = "LATENCY"; + private String sequencerType = "THROUGHPUT"; private int feedTaskLimit = 1000; - private int feedMasterTaskLimit = 0; + private int feedMasterTaskLimit = 1000; private String sharedFieldWriterExecutor = "NONE"; private boolean firstTimeDeployment = false; private String responseSequencerType = "ADAPTIVE"; @@ -57,9 +57,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; private int maxConcurrentMergesPerNode = 16; - private int maxMergeQueueSize = 1024; - private boolean ignoreMergeQueueLimit = false; - private int largeRankExpressionLimit = 8192; + private int maxMergeQueueSize = 100; + private boolean ignoreMergeQueueLimit = true; private boolean allowDisableMtls = true; private List<X509Certificate> operatorCertificates = Collections.emptyList(); private double resourceLimitDisk = 0.8; @@ -67,17 +66,15 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double minNodeRatioPerGroup = 0.0; private boolean containerDumpHeapOnShutdownTimeout = false; private double containerShutdownTimeout = 50.0; - private int distributorMergeBusyWait = 10; - private int docstoreCompressionLevel = 9; + private int distributorMergeBusyWait = 1; private int maxUnCommittedMemory = 123456; - private double diskBloatFactor = 0.2; - private boolean distributorEnhancedMaintenanceScheduling = false; - private boolean asyncApplyBucketDiff = false; - private boolean unorderedMergeChaining = false; + private boolean distributorEnhancedMaintenanceScheduling = true; + private boolean asyncApplyBucketDiff = true; + private boolean unorderedMergeChaining = true; private List<String> zoneDnsSuffixes = List.of(); private int maxCompactBuffers = 1; private boolean failDeploymentWithInvalidJvmOptions = false; - private double tlsSizeFraction = 0.07; + private String persistenceAsyncThrottling = "UNLIMITED"; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -115,7 +112,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @Override public boolean allowDisableMtls() { return allowDisableMtls; } @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } - @Override public int largeRankExpressionLimit() { return largeRankExpressionLimit; } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; } @@ -126,8 +122,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } - @Override public double diskBloatFactor() { return diskBloatFactor; } - @Override public int docstoreCompressionLevel() { return docstoreCompressionLevel; } @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } @@ -135,23 +129,13 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } - @Override public double tlsSizeFraction() { return tlsSizeFraction; } + @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; return this; } - public TestProperties docstoreCompressionLevel(int docstoreCompressionLevel) { - this.docstoreCompressionLevel = docstoreCompressionLevel; - return this; - } - - public TestProperties diskBloatFactor(double diskBloatFactor) { - this.diskBloatFactor = diskBloatFactor; - return this; - } - public TestProperties containerDumpHeapOnShutdownTimeout(boolean value) { containerDumpHeapOnShutdownTimeout = value; return this; @@ -160,10 +144,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea containerShutdownTimeout = value; return this; } - public TestProperties largeRankExpressionLimit(int value) { - largeRankExpressionLimit = value; - return this; - } + public TestProperties setFeedConcurrency(double feedConcurrency) { this.feedConcurrency = feedConcurrency; return this; @@ -356,8 +337,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties tlsSizeFraction(double tlsSizeFraction) { - this.tlsSizeFraction = tlsSizeFraction; + public TestProperties setPersistenceAsyncThrottling(String type) { + this.persistenceAsyncThrottling = type; return this; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java index 3fdea71da2c..a1299c12307 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java @@ -10,9 +10,14 @@ import java.util.concurrent.ConcurrentHashMap; public class LargeRankExpressions { private final Map<String, RankExpressionBody> expressions = new ConcurrentHashMap<>(); private final FileRegistry fileRegistry; + private final int limit; public LargeRankExpressions(FileRegistry fileRegistry) { + this(fileRegistry, 8192); + } + public LargeRankExpressions(FileRegistry fileRegistry, int limit) { this.fileRegistry = fileRegistry; + this.limit = limit; } public void add(RankExpressionBody expression) { @@ -29,6 +34,7 @@ public class LargeRankExpressions { } } } + public int limit() { return limit; } /** Returns a read-only map of the ranking constants in this indexed by name */ public Map<String, RankExpressionBody> asMap() { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index a33498a37ec..d7bcd295f09 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -140,7 +140,6 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final int numSearchPartitions; private final double termwiseLimit; private final double rankScoreDropLimit; - private final int largeRankExpressionLimit; /** * The rank type definitions used to derive settings for the native rank features @@ -176,7 +175,6 @@ public class RawRankProfile implements RankProfilesConfig.Producer { keepRankCount = compiled.getKeepRankCount(); rankScoreDropLimit = compiled.getRankScoreDropLimit(); ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); - largeRankExpressionLimit = deployProperties.featureFlags().largeRankExpressionLimit(); rankProperties = new ArrayList<>(compiled.getRankProperties()); Map<String, RankProfile.RankingExpressionFunction> functions = compiled.getFunctions(); @@ -419,7 +417,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (ListIterator<Pair<String, String>> iter = properties.listIterator(); iter.hasNext();) { Pair<String, String> property = iter.next(); String expression = property.getSecond(); - if (expression.length() > largeRankExpressionLimit) { + if (expression.length() > largeRankExpressions.limit()) { String propertyName = property.getFirst(); String functionName = RankingExpression.extractScriptName(propertyName); if (functionName != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java index 6e6f027b520..0b99496a9b4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerCluster.java @@ -50,14 +50,12 @@ public class ClusterControllerCluster extends AbstractConfigProducer<ClusterCont public void getConfig(ZookeeperServerConfig.Builder builder) { builder.clientPort(ZK_CLIENT_PORT); builder.juteMaxBuffer(1024 * 1024); // 1 Mb should be more than enough for cluster controller - boolean oldQuorumExists = containerCluster.getContainers().stream() // More than half the previous hosts must be present in the new config for quorum to persist. - .filter(container -> previousHosts.contains(container.getHostName())) // Set intersection is symmetric. - .count() > previousHosts.size() / 2; for (ClusterControllerContainer container : containerCluster.getContainers()) { ZookeeperServerConfig.Server.Builder serverBuilder = new ZookeeperServerConfig.Server.Builder(); serverBuilder.hostname(container.getHostName()); serverBuilder.id(container.index()); - serverBuilder.joining(oldQuorumExists && ! previousHosts.contains(container.getHostName())); + serverBuilder.joining( ! previousHosts.isEmpty() && ! previousHosts.contains(container.getHostName())); + serverBuilder.retired(container.isRetired()); builder.server(serverBuilder); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java new file mode 100644 index 00000000000..87a84911d3e --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/BundleValidator.java @@ -0,0 +1,224 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import aQute.bnd.header.Parameters; +import aQute.bnd.osgi.Domain; +import aQute.bnd.version.VersionRange; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.path.Path; +import com.yahoo.vespa.model.VespaModel; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathFactory; +import java.io.IOException; +import java.io.StringReader; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.jar.Attributes; +import java.util.jar.JarFile; +import java.util.jar.Manifest; +import java.util.logging.Level; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * A validator for bundles. Uses BND library for some of the validation. + * + * @author hmusum + * @author bjorncs + */ +public class BundleValidator extends Validator { + + public BundleValidator() {} + + @Override + public void validate(VespaModel model, DeployState deployState) { + ApplicationPackage app = deployState.getApplicationPackage(); + for (ComponentInfo info : app.getComponentsInfo(deployState.getVespaVersion())) { + Path path = Path.fromString(info.getPathRelativeToAppDir()); + try { + DeployLogger deployLogger = deployState.getDeployLogger(); + deployLogger.log(Level.FINE, String.format("Validating bundle at '%s'", path)); + JarFile jarFile = new JarFile(app.getFileReference(path)); + validateJarFile(deployLogger, jarFile); + } catch (IOException e) { + throw new IllegalArgumentException( + "Failed to validate JAR file '" + path.last() + "'", e); + } + } + } + + void validateJarFile(DeployLogger deployLogger, JarFile jarFile) throws IOException { + Manifest manifest = jarFile.getManifest(); + String filename = Paths.get(jarFile.getName()).getFileName().toString(); + if (manifest == null) { + throw new IllegalArgumentException("Non-existing or invalid manifest in " + filename); + } + validateManifest(deployLogger, filename, manifest); + getPomXmlContent(deployLogger, jarFile) + .ifPresent(pomXml -> validatePomXml(deployLogger, filename, pomXml)); + } + + private void validateManifest(DeployLogger deployLogger, String filename, Manifest mf) { + // Check for required OSGI headers + Attributes attributes = mf.getMainAttributes(); + HashSet<String> mfAttributes = new HashSet<>(); + for (Map.Entry<Object,Object> entry : attributes.entrySet()) { + mfAttributes.add(entry.getKey().toString()); + } + List<String> requiredOSGIHeaders = Arrays.asList( + "Bundle-ManifestVersion", "Bundle-Name", "Bundle-SymbolicName", "Bundle-Version"); + for (String header : requiredOSGIHeaders) { + if (!mfAttributes.contains(header)) { + throw new IllegalArgumentException("Required OSGI header '" + header + + "' was not found in manifest in '" + filename + "'"); + } + } + + if (attributes.getValue("Bundle-Version").endsWith(".SNAPSHOT")) { + deployLogger.logApplicationPackage(Level.WARNING, "Deploying snapshot bundle " + filename + + ".\nTo use this bundle, you must include the qualifier 'SNAPSHOT' in the version specification in services.xml."); + } + + if (attributes.getValue("Import-Package") != null) { + validateImportedPackages(deployLogger, filename, mf); + } + } + + private static void validateImportedPackages(DeployLogger deployLogger, String filename, Manifest manifest) { + Domain osgiHeaders = Domain.domain(manifest); + Parameters importPackage = osgiHeaders.getImportPackage(); + Map<DeprecatedProvidedBundle, List<String>> deprecatedPackagesInUse = new HashMap<>(); + + importPackage.forEach((packageName, attrs) -> { + VersionRange versionRange = attrs.getVersion() != null + ? VersionRange.parseOSGiVersionRange(attrs.getVersion()) + : null; + + for (DeprecatedProvidedBundle deprecatedBundle : DeprecatedProvidedBundle.values()) { + for (Predicate<String> matcher : deprecatedBundle.javaPackageMatchers) { + if (matcher.test(packageName) + && (versionRange == null || deprecatedBundle.versionDiscriminator.test(versionRange))) { + deprecatedPackagesInUse.computeIfAbsent(deprecatedBundle, __ -> new ArrayList<>()) + .add(packageName); + } + } + } + }); + + deprecatedPackagesInUse.forEach((artifact, packagesInUse) -> { + deployLogger.logApplicationPackage(Level.WARNING, + String.format("For JAR file '%s': \n" + + "Manifest imports the following Java packages from '%s': %s. \n" + + "%s", + filename, artifact.name, packagesInUse, artifact.description)); + }); + } + + private static final Pattern POM_FILE_LOCATION = Pattern.compile("META-INF/maven/.+?/.+?/pom.xml"); + + private Optional<String> getPomXmlContent(DeployLogger deployLogger, JarFile jarFile) { + return jarFile.stream() + .filter(f -> POM_FILE_LOCATION.matcher(f.getName()).matches()) + .findFirst() + .map(f -> { + try { + return new String(jarFile.getInputStream(f).readAllBytes()); + } catch (IOException e) { + deployLogger.log(Level.INFO, + String.format("Unable to read '%s' from '%s'", f.getName(), jarFile.getName())); + return null; + } + }); + } + + private void validatePomXml(DeployLogger deployLogger, String jarFilename, String pomXmlContent) { + try { + Document pom = DocumentBuilderFactory.newDefaultInstance().newDocumentBuilder() + .parse(new InputSource(new StringReader(pomXmlContent))); + NodeList dependencies = (NodeList) XPathFactory.newDefaultInstance().newXPath() + .compile("/project/dependencies/dependency") + .evaluate(pom, XPathConstants.NODESET); + for (int i = 0; i < dependencies.getLength(); i++) { + Element dependency = (Element) dependencies.item(i); + String groupId = dependency.getElementsByTagName("groupId").item(0).getTextContent(); + String artifactId = dependency.getElementsByTagName("artifactId").item(0).getTextContent(); + for (DeprecatedMavenArtifact deprecatedArtifact : DeprecatedMavenArtifact.values()) { + if (groupId.equals(deprecatedArtifact.groupId) && artifactId.equals(deprecatedArtifact.artifactId)) { + deployLogger.logApplicationPackage(Level.WARNING, + String.format( + "The pom.xml of bundle '%s' includes a dependency to the artifact '%s:%s'. \n%s", + jarFilename, groupId, artifactId, deprecatedArtifact.description)); + } + } + } + } catch (ParserConfigurationException e) { + throw new RuntimeException(e); + } catch (Exception e) { + deployLogger.log(Level.INFO, String.format("Unable to parse pom.xml from %s", jarFilename)); + } + } + + private enum DeprecatedMavenArtifact { + VESPA_HTTP_CLIENT_EXTENSION("com.yahoo.vespa", "vespa-http-client-extensions", + "This artifact will be removed in Vespa 8. " + + "Programmatic use can be safely removed from system/staging tests. " + + "See internal Vespa 8 release notes for details."); + + final String groupId; + final String artifactId; + final String description; + + DeprecatedMavenArtifact(String groupId, String artifactId, String description) { + this.groupId = groupId; + this.artifactId = artifactId; + this.description = description; + } + } + + private enum DeprecatedProvidedBundle { + ORG_JSON("org.json:json", + "The org.json library will no longer provided by jdisc runtime on Vespa 8. " + + "See https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime.", + Set.of("org\\.json")); + + final String name; + final Collection<Predicate<String>> javaPackageMatchers; + final Predicate<VersionRange> versionDiscriminator; + final String description; + + DeprecatedProvidedBundle(String name, String description, Collection<String> javaPackagePatterns) { + this(name, description, __ -> true, javaPackagePatterns); + } + + DeprecatedProvidedBundle(String name, + String description, + Predicate<VersionRange> versionDiscriminator, + Collection<String> javaPackagePatterns) { + this.name = name; + this.javaPackageMatchers = javaPackagePatterns.stream() + .map(s -> Pattern.compile(s).asMatchPredicate()) + .collect(Collectors.toList()); + this.versionDiscriminator = versionDiscriminator; + this.description = description; + } + } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComponentValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComponentValidator.java deleted file mode 100644 index 21e396959a7..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComponentValidator.java +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.path.Path; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.config.application.api.ComponentInfo; -import com.yahoo.config.application.api.DeployLogger; -import java.io.IOException; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.jar.Attributes; -import java.util.jar.JarFile; -import java.util.jar.Manifest; -import java.util.logging.Level; -import java.util.zip.ZipException; - -/** - * A validator for bundles. Uses BND library for some of the validation (not active yet) - * - * @author hmusum - * @since 2010-11-11 - */ -public class ComponentValidator extends Validator { - private JarFile jarFile; - - public ComponentValidator() { - } - - public ComponentValidator(JarFile jarFile) { - this.jarFile = jarFile; - } - - @Override - public void validate(VespaModel model, DeployState deployState) { - ApplicationPackage app = deployState.getApplicationPackage(); - for (ComponentInfo info : app.getComponentsInfo(deployState.getVespaVersion())) { - try { - this.jarFile = new JarFile(app.getFileReference(Path.fromString(info.getPathRelativeToAppDir()))); - } catch (ZipException e) { - throw new IllegalArgumentException("Error opening jar file '" + info.getPathRelativeToAppDir() + - "'. Please check that this is a valid jar file"); - } catch (IOException e) { - e.printStackTrace(); - } - try { - validateAll(deployState.getDeployLogger()); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - public void validateAll(DeployLogger deployLogger) throws IOException { - validateOSGIHeaders(deployLogger); - } - - public void validateOSGIHeaders(DeployLogger deployLogger) throws IOException { - Manifest mf = jarFile.getManifest(); - if (mf == null) { - throw new IllegalArgumentException("Non-existing or invalid manifest in " + jarFile.getName()); - } - - // Check for required OSGI headers - Attributes attributes = mf.getMainAttributes(); - HashSet<String> mfAttributes = new HashSet<>(); - for (Object attributeSet : attributes.entrySet()) { - Map.Entry<Object, Object> e = (Map.Entry<Object, Object>) attributeSet; - mfAttributes.add(e.getKey().toString()); - } - List<String> requiredOSGIHeaders = Arrays.asList( - "Bundle-ManifestVersion", "Bundle-Name", "Bundle-SymbolicName", "Bundle-Version"); - for (String header : requiredOSGIHeaders) { - if (!mfAttributes.contains(header)) { - throw new IllegalArgumentException("Required OSGI header '" + header + - "' was not found in manifest in '" + jarFile.getName() + "'"); - } - } - - if (attributes.getValue("Bundle-Version").endsWith(".SNAPSHOT")) { - deployLogger.logApplicationPackage(Level.WARNING, "Deploying snapshot bundle " + jarFile.getName() + - ".\nTo use this bundle, you must include the qualifier 'SNAPSHOT' in the version specification in services.xml."); - } - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 5215fdcb301..08dc73a1bd0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -63,7 +63,7 @@ public class Validation { new RoutingSelectorValidator().validate(model, deployState); } new SchemasDirValidator().validate(model, deployState); - new ComponentValidator().validate(model, deployState); + new BundleValidator().validate(model, deployState); new SearchDataTypeValidator().validate(model, deployState); new ComplexAttributeFieldsValidator().validate(model, deployState); new StreamingValidator().validate(model, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 02d5bf719cf..5aaba9550d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -47,12 +47,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { if ( ! admin.multitenant()) admin.setClusterControllers(addConfiguredClusterControllers(deployState, admin, adminE), deployState); - ModelElement adminElement = new ModelElement(adminE); - addLogForwarders(adminElement.child("logforwarding"), admin); - - if (adminElement.child("filedistribution") != null) { - deployState.getDeployLogger().logApplicationPackage(Level.WARNING, "'filedistribution' element is deprecated and ignored"); - } + addLogForwarders(new ModelElement(adminE).child("logforwarding"), admin); } private List<Configserver> parseConfigservers(DeployState deployState, Admin admin, Element adminE) { @@ -117,7 +112,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { configserverE = XML.getChild(adminE, "adminserver"); else deployState.getDeployLogger().logApplicationPackage(Level.INFO, - "Specifying configserver without parent element configservers in services.xml is deprecated"); + "Specifying configserver without parent element configservers in services.xml is deprecated and will be removed in Vespa 8"); return List.of(new ConfigserverBuilder(0, configServerSpecs).build(deployState, configServers, configserverE)); } else { @@ -158,7 +153,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { } @Override - protected Logserver doBuild(DeployState deployState, AbstractConfigProducer parent, Element producerSpec) { + protected Logserver doBuild(DeployState deployState, AbstractConfigProducer<?> parent, Element producerSpec) { return new Logserver(parent); } } @@ -177,7 +172,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { } @Override - protected Configserver doBuild(DeployState deployState, AbstractConfigProducer parent, Element spec) { + protected Configserver doBuild(DeployState deployState, AbstractConfigProducer<?> parent, Element spec) { var configServer = new Configserver(parent, "configserver." + i, rpcPort); configServer.setProp("index", i); return configServer; @@ -193,7 +188,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { } @Override - protected Slobrok doBuild(DeployState deployState, AbstractConfigProducer parent, Element spec) { + protected Slobrok doBuild(DeployState deployState, AbstractConfigProducer<?> parent, Element spec) { return new Slobrok(parent, i, deployState.featureFlags()); } @@ -209,7 +204,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { } @Override - protected ClusterControllerContainer doBuild(DeployState deployState, AbstractConfigProducer parent, Element spec) { + protected ClusterControllerContainer doBuild(DeployState deployState, AbstractConfigProducer<?> parent, Element spec) { return new ClusterControllerContainer(parent, i, runStandaloneZooKeeper, deployState, false); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index c4d420f2d44..438e143bdfd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -309,8 +309,9 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat ZookeeperServerConfig.Server.Builder serverBuilder = new ZookeeperServerConfig.Server.Builder(); serverBuilder.hostname(container.getHostName()) .id(container.index()) - .joining(!previousHosts.isEmpty() && - !previousHosts.contains(container.getHostName())); + .joining( ! previousHosts.isEmpty() && + ! previousHosts.contains(container.getHostName())) + .retired(container.isRetired()); builder.server(serverBuilder); builder.dynamicReconfiguration(true); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 8ac30f66ae7..302e8eff2d8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -47,7 +47,7 @@ public class ConfigserverCluster extends AbstractConfigProducer this.containerCluster = containerCluster; // If we are in a config server cluster the correct zone is propagated through cloud config options, - // not through config to deployment options (see StandaloneContainerApplication.scala), + // not through config to deployment options (see StandaloneContainerApplication.java), // so we need to propagate the zone options into the container from here Environment environment = options.environment().isPresent() ? Environment.from(options.environment().get()) : Environment.defaultEnvironment(); RegionName region = options.region().isPresent() ? RegionName.from(options.region().get()) : RegionName.defaultName(); @@ -83,6 +83,8 @@ public class ConfigserverCluster extends AbstractConfigProducer if (options.zookeeperClientPort().isPresent()) { builder.clientPort(options.zookeeperClientPort().get()); } + + builder.snapshotMethod(options.zooKeeperSnapshotMethod()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index fe3bd271f2f..c61c140c05b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -38,4 +38,5 @@ public interface CloudConfigOptions { Optional<String> loadBalancerAddress(); Optional<String> athenzDnsSuffix(); Optional<String> ztsUrl(); + String zooKeeperSnapshotMethod(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 657eb6a29e7..3bb1a9d2bf7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -1061,15 +1061,22 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return CONTAINER_TAG.equals(element.getTagName()) || DEPRECATED_CONTAINER_TAG.equals(element.getTagName()); } - private static class JvmOptions { + /** + * Validates JVM options and logs a warning or fails deployment (depending on feature flag) + * if anyone of them has invalid syntax or is an option that is unsupported for the running system. + */ + private static class JvmOptions { - private static final Pattern validPattern = Pattern.compile("-[a-zA-z0-9=:./]+"); + private static final Pattern validPattern = Pattern.compile("-[a-zA-z0-9=:./,+-]+"); + // debug port will not be available in hosted, don't allow + private static final Pattern invalidInHostedatttern = Pattern.compile("-Xrunjdwp:transport=.*"); private final ContainerCluster<?> cluster; private final Element nodesElement; private final DeployLogger logger; private final boolean legacyOptions; private final boolean failDeploymentWithInvalidJvmOptions; + private final boolean isHosted; public JvmOptions(ContainerCluster<?> cluster, Element nodesElement, DeployState deployState, boolean legacyOptions) { this.cluster = cluster; @@ -1077,6 +1084,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { this.logger = deployState.getDeployLogger(); this.legacyOptions = legacyOptions; this.failDeploymentWithInvalidJvmOptions = deployState.featureFlags().failDeploymentWithInvalidJvmOptions(); + this.isHosted = deployState.isHosted(); } String build() { @@ -1086,7 +1094,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Element jvmElement = XML.getChild(nodesElement, "jvm"); if (jvmElement == null) return ""; String jvmOptions = jvmElement.getAttribute(VespaDomBuilder.OPTIONS); - if (jvmOptions == null) return ""; + if (jvmOptions.isEmpty()) return ""; validateJvmOptions(jvmOptions); return jvmOptions; } @@ -1135,10 +1143,18 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .filter(option -> !Pattern.matches(validPattern.pattern(), option)) .sorted() .collect(Collectors.toList()); + if (isHosted) + invalidOptions.addAll(Arrays.stream(optionList) + .filter(option -> !option.isEmpty()) + .filter(option -> Pattern.matches(invalidInHostedatttern.pattern(), option)) + .sorted() + .collect(Collectors.toList())); if (invalidOptions.isEmpty()) return; - String message = "Invalid JVM options in services.xml: " + String.join(",", invalidOptions); + String message = "Invalid or misplaced JVM options in services.xml: " + + String.join(",", invalidOptions) + "." + + " See https://docs.vespa.ai/en/reference/services-container.html#jvm"; if (failDeploymentWithInvalidJvmOptions) throw new IllegalArgumentException(message); else @@ -1147,8 +1163,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } /** - * Validates JVM GC options and logs a warning if anyone of them has invalid syntax or is an option - * that is unsupported for the running system (e.g. uses CMS options for hosted Vespa, which uses JDK 17) + * Validates JVM GC options and logs a warning or fails deployment (depending on feature flag) + * if anyone of them has invalid syntax or is an option that is unsupported for the running system + * (e.g. uses CMS options for hosted Vespa, which uses JDK 17). */ private static class JvmGcOptions { @@ -1201,7 +1218,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (options.isEmpty()) return; Collections.sort(options); - String message = "Invalid JVM GC options in services.xml: " + String.join(",", options); + String message = "Invalid or misplaced JVM GC options in services.xml: " + + String.join(",", options) + "." + + " See https://docs.vespa.ai/en/reference/services-container.html#jvm"; if (failDeploymentWithInvalidJvmOptions) throw new IllegalArgumentException(message); else diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 54d09bacfa9..d0cba617cfc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -43,6 +43,9 @@ import static java.util.stream.Collectors.toList; */ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> implements ProtonConfig.Producer, DispatchConfig.Producer { + private static final int DEFAULT_DOC_STORE_COMPRESSION_LEVEL = 3; + private static final double DEFAULT_DISK_BLOAT = 0.25; + private final boolean flushOnShutdown; private final Boolean syncTransactionLog; @@ -68,8 +71,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final int feedMasterTaskLimit; private final ProtonConfig.Feeding.Shared_field_writer_executor.Enum sharedFieldWriterExecutor; private final double defaultFeedConcurrency; - private final double defaultDiskBloatFactor; - private final int defaultDocStoreCompressionLevel; private final boolean forwardIssuesToQrs; private final int defaultMaxCompactBuffers; @@ -223,8 +224,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.feedMasterTaskLimit = featureFlags.feedMasterTaskLimit(); this.sharedFieldWriterExecutor = convertSharedFieldWriterExecutor(featureFlags.sharedFieldWriterExecutor()); this.defaultFeedConcurrency = featureFlags.feedConcurrency(); - this.defaultDiskBloatFactor = featureFlags.diskBloatFactor(); - this.defaultDocStoreCompressionLevel = featureFlags.docstoreCompressionLevel(); this.forwardIssuesToQrs = featureFlags.forwardIssuesAsErrors(); this.defaultMaxCompactBuffers = featureFlags.maxCompactBuffers(); } @@ -291,7 +290,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> if (element == null) { searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags().tlsSizeFraction()); + fractionOfMemoryReserved); searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState.getDeployLogger()); @@ -423,10 +422,10 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } else { builder.feeding.concurrency(defaultFeedConcurrency); } - builder.flush.memory.diskbloatfactor(defaultDiskBloatFactor); - builder.flush.memory.each.diskbloatfactor(defaultDiskBloatFactor); - builder.summary.log.chunk.compression.level(defaultDocStoreCompressionLevel); - builder.summary.log.compact.compression.level(defaultDocStoreCompressionLevel); + builder.flush.memory.diskbloatfactor(DEFAULT_DISK_BLOAT); + builder.flush.memory.each.diskbloatfactor(DEFAULT_DISK_BLOAT); + builder.summary.log.chunk.compression.level(DEFAULT_DOC_STORE_COMPRESSION_LEVEL); + builder.summary.log.compact.compression.level(DEFAULT_DOC_STORE_COMPRESSION_LEVEL); builder.forward_issues(forwardIssuesToQrs); int numDocumentDbs = builder.documentdb.size(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index d7f6fb6c581..1f3a76b766e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -46,6 +46,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final ContentCluster cluster; private final int reponseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; + private final StorFilestorConfig.Async_operation_throttler_type.Enum asyncOperationThrottlerType; private final boolean useAsyncMessageHandlingOnSchedule; private final boolean asyncApplyBucketDiff; @@ -57,11 +58,20 @@ public class FileStorProducer implements StorFilestorConfig.Producer { } } + private static StorFilestorConfig.Async_operation_throttler_type.Enum toAsyncOperationThrottlerType(String throttlerType) { + try { + return StorFilestorConfig.Async_operation_throttler_type.Enum.valueOf(throttlerType); + } catch (Throwable t) { + return StorFilestorConfig.Async_operation_throttler_type.UNLIMITED; + } + } + public FileStorProducer(ModelContext.FeatureFlags featureFlags, ContentCluster parent, Integer numThreads) { this.numThreads = numThreads; this.cluster = parent; this.reponseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); + this.asyncOperationThrottlerType = toAsyncOperationThrottlerType(featureFlags.persistenceAsyncThrottling()); useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); asyncApplyBucketDiff = featureFlags.asyncApplyBucketDiff(); } @@ -76,6 +86,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); builder.async_apply_bucket_diff(asyncApplyBucketDiff); + builder.async_operation_throttler_type(asyncOperationThrottlerType); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 0eb0bd0bf2d..9b9a525ab29 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -15,8 +15,9 @@ import static java.lang.Long.max; public class NodeResourcesTuning implements ProtonConfig.Producer { private final static double SUMMARY_FILE_SIZE_AS_FRACTION_OF_MEMORY = 0.02; - private final static double SUMMARY_CACHE_SIZE_AS_FRACTION_OF_MEMORY = 0.05; - private final static double MEMORY_GAIN_AS_FRACTION_OF_MEMORY = 0.10; + private final static double SUMMARY_CACHE_SIZE_AS_FRACTION_OF_MEMORY = 0.04; + private final static double MEMORY_GAIN_AS_FRACTION_OF_MEMORY = 0.08; + private final static double TLS_SIZE_FRACTION = 0.02; final static long MB = 1024 * 1024; public final static long GB = MB * 1024; // This is an approximate number base on observation of a node using 33G memory with 765M docs @@ -24,19 +25,16 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final NodeResources resources; private final int threadsPerSearch; private final double fractionOfMemoryReserved; - private final double tlsSizeFraction; // "Reserve" 0.5GB of memory for other processes running on the content node (config-proxy, metrics-proxy). public static final double reservedMemoryGb = 0.5; public NodeResourcesTuning(NodeResources resources, int threadsPerSearch, - double fractionOfMemoryReserved, - double tlsSizeFraction) { + double fractionOfMemoryReserved) { this.resources = resources; this.threadsPerSearch = threadsPerSearch; this.fractionOfMemoryReserved = fractionOfMemoryReserved; - this.tlsSizeFraction = tlsSizeFraction; } @Override @@ -93,7 +91,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) { - long tlsSizeBytes = (long) ((resources.diskGb() * tlsSizeFraction) * GB); + long tlsSizeBytes = (long) ((resources.diskGb() * TLS_SIZE_FRACTION) * GB); tlsSizeBytes = max(2*GB, min(tlsSizeBytes, 100 * GB)); builder.maxtlssize(tlsSizeBytes); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 31513a273b2..28d1fbe72ef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -60,7 +60,7 @@ public class SearchNode extends AbstractService implements private final boolean isHostedVespa; private final boolean flushOnShutdown; - private NodeSpec nodeSpec; + private final NodeSpec nodeSpec; private int distributionKey; private final String clusterName; private TransactionLogServer tls; @@ -68,7 +68,6 @@ public class SearchNode extends AbstractService implements private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; private final double fractionOfMemoryReserved; - private final double tlsSizeFraction; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<SearchNode> { @@ -97,8 +96,7 @@ public class SearchNode extends AbstractService implements @Override protected SearchNode doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { return new SearchNode(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, - flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), fractionOfMemoryReserved, - deployState.featureFlags().tlsSizeFraction()); + flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), fractionOfMemoryReserved); } } @@ -106,16 +104,16 @@ public class SearchNode extends AbstractService implements public static SearchNode create(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved, double tlsSizeFraction) { + double fractionOfMemoryReserved) { return new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown, - tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, tlsSizeFraction); + tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved); } private SearchNode(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved, double tlsSizeFraction) { - this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, tlsSizeFraction); + double fractionOfMemoryReserved) { + this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; setPropertiesElastic(clusterName, distributionKey); @@ -123,12 +121,11 @@ public class SearchNode extends AbstractService implements private SearchNode(AbstractConfigProducer parent, String name, NodeSpec nodeSpec, String clusterName, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved, double tlsSizeFraction) { + double fractionOfMemoryReserved) { super(parent, name); setOmpNumThreads(1); this.isHostedVespa = isHostedVespa; this.fractionOfMemoryReserved = fractionOfMemoryReserved; - this.tlsSizeFraction = tlsSizeFraction; this.nodeSpec = nodeSpec; this.clusterName = clusterName; this.flushOnShutdown = flushOnShutdown; @@ -282,7 +279,7 @@ public class SearchNode extends AbstractService implements if (nodeResources.isPresent()) { var nodeResourcesTuning = new NodeResourcesTuning(nodeResources.get(), tuning.map(Tuning::threadsPerSearch).orElse(1), - fractionOfMemoryReserved, tlsSizeFraction); + fractionOfMemoryReserved); nodeResourcesTuning.getConfig(builder); tuning.ifPresent(t -> t.getConfig(builder)); diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 51a286a13c8..819e6b79fbb 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -58,8 +58,10 @@ Upgrade = element upgrade { BlockChange = element block-change { attribute revision { xsd:boolean }? & attribute version { xsd:boolean }? & - attribute days { xsd:string } & - attribute hours { xsd:string } & + attribute days { xsd:string }? & + attribute hours { xsd:string }? & + attribute from-date { xsd:string }? & + attribute to-date { xsd:string }? & attribute time-zone { xsd:string }? } diff --git a/config-model/src/main/resources/schema/services.rnc b/config-model/src/main/resources/schema/services.rnc index 758fa107ee8..c8467898639 100644 --- a/config-model/src/main/resources/schema/services.rnc +++ b/config-model/src/main/resources/schema/services.rnc @@ -12,6 +12,7 @@ include "legacygenericcluster.rnc" start = element services { attribute version { "1.0" }? & attribute application-type { "hosted-infrastructure" }? & + element legacy { element v7-geo-positions { xsd:boolean } }? & LegacyGenericCluster* & GenericCluster* & GenericConfig* & diff --git a/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..760a9ecf00f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/import-warnings/META-INF/MANIFEST.MF @@ -0,0 +1,10 @@ +Manifest-Version: 1.0 +Export-Package: com.yahoo.vespa.test.myapp;version=1.0.0 +Bundle-ManifestVersion: 2 +Bundle-SymbolicName: my-bundle +Bundle-Version: 7.0.0 +Created-By: vespa container maven plugin +Bundle-Name: my-bundle +Bundle-Vendor: Yahoo! +Import-Package: org.json;version="[0.0.0,1)" + diff --git a/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar b/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar Binary files differdeleted file mode 100644 index 84781c4802e..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/missing_osgi_headers.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar b/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar Binary files differdeleted file mode 100644 index f4f7dd4e127..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/nomanifest.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd new file mode 100644 index 00000000000..c52570face3 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/base.sd @@ -0,0 +1,7 @@ +search base { + document base { + field base type string { + indexing: summary | index + } + } +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd new file mode 100644 index 00000000000..73b540627d7 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/book.sd @@ -0,0 +1,184 @@ +search book { + document book inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field dispauthor type string { + bolding: on + index-to: default, dispauthor + indexing: index|summary + rank-type: about + } + field author type string { + bolding: on + index-to: default, author + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field isbn type string { + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field series type string { + index-to: default, series + indexing: index + rank-type: about + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field price type string { + indexing: summary + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field mid type int { + indexing: attribute|summary|collapse + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field fmt type string { + indexing: index|summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 6 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field formats type string { + indexing: index|summary + } + field age type string { + indexing: index|summary + } + field sales type int { + indexing: attribute|summary + } + field more_url type string { + indexing: summary + } + field more_price type string { + indexing: summary + } + field more_format type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field userrate type int { + indexing: attribute|summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute: no-update + rank-type: about + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 6 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 6 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { attribute; } | summary; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd new file mode 100644 index 00000000000..498bc79489f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/music.sd @@ -0,0 +1,12 @@ +search music { + document music inherits base { + field f1 type string { + indexing: summary | index + index-to: f1, all + } + field f2 type string { + indexing: summary | index + index-to: f2, all + } + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd new file mode 100644 index 00000000000..b010b6d9769 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/nomanifest/searchdefinitions/video.sd @@ -0,0 +1,182 @@ +search video { + document video inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field director type string { + bolding: on + index-to: default, director + indexing: index|summary + rank-type: about + } + field disp_actor type string { + bolding: on + index-to: default, disp_actor + indexing: index|summary + rank-type: about + } + field actor type string { + bolding: on + index-to: default, actor + indexing: index|summary + rank-type: about + } + field fmt type string { + index-to: default, fmt + indexing: index|summary + rank-type: about + } + field isbn type string { + bolding: on + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field mid type int { + indexing: attribute|summary|collapse + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 10 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field sales type int { + indexing: attribute|summary + } + field surl type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field ew type string { + indexing: index|summary + rank-type: about + } + field ed type string { + indexing: summary + } + field userrate type int { + indexing: summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute : no-update + rank-type: about + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 10 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 10 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { + attribute; + }; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok.jar b/config-model/src/test/cfg/application/validation/testjars/ok.jar Binary files differdeleted file mode 100644 index fce043c6ff7..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/ok.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..53773b8b1cc --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/META-INF/MANIFEST.MF @@ -0,0 +1,7 @@ +Manifest-Version: 1.0 +Created-By: 1.6.0_20 (Apple Inc.) +Bundle-ManifestVersion: 2 +Bundle-Name: ok +Bundle-SymbolicName: ok +Bundle-Version: 0 + diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd new file mode 100644 index 00000000000..c52570face3 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/base.sd @@ -0,0 +1,7 @@ +search base { + document base { + field base type string { + indexing: summary | index + } + } +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd new file mode 100644 index 00000000000..73b540627d7 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/book.sd @@ -0,0 +1,184 @@ +search book { + document book inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field dispauthor type string { + bolding: on + index-to: default, dispauthor + indexing: index|summary + rank-type: about + } + field author type string { + bolding: on + index-to: default, author + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field isbn type string { + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field series type string { + index-to: default, series + indexing: index + rank-type: about + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field price type string { + indexing: summary + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field mid type int { + indexing: attribute|summary|collapse + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field fmt type string { + indexing: index|summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 6 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field formats type string { + indexing: index|summary + } + field age type string { + indexing: index|summary + } + field sales type int { + indexing: attribute|summary + } + field more_url type string { + indexing: summary + } + field more_price type string { + indexing: summary + } + field more_format type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field userrate type int { + indexing: attribute|summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute: no-update + rank-type: about + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 6 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 6 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { attribute; } | summary; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd new file mode 100644 index 00000000000..498bc79489f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/music.sd @@ -0,0 +1,12 @@ +search music { + document music inherits base { + field f1 type string { + indexing: summary | index + index-to: f1, all + } + field f2 type string { + indexing: summary | index + index-to: f2, all + } + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd new file mode 100644 index 00000000000..b010b6d9769 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/ok/searchdefinitions/video.sd @@ -0,0 +1,182 @@ +search video { + document video inherits base { + field title type string { + bolding: on + index-to: default, title + indexing: index|summary + rank-type: about + } + field keys type string { + index-to: default, keys + indexing: index + rank-type: about + } + field director type string { + bolding: on + index-to: default, director + indexing: index|summary + rank-type: about + } + field disp_actor type string { + bolding: on + index-to: default, disp_actor + indexing: index|summary + rank-type: about + } + field actor type string { + bolding: on + index-to: default, actor + indexing: index|summary + rank-type: about + } + field fmt type string { + index-to: default, fmt + indexing: index|summary + rank-type: about + } + field isbn type string { + bolding: on + index-to: default, isbn + indexing: index|summary + rank-type: about + } + field mid type int { + indexing: attribute|summary|collapse + } + field url type string { + indexing: summary + } + field image type string { + indexing: summary + } + field img85 type string { + indexing: summary + } + field img110 type string { + indexing: summary + } + field limg type string { + indexing: summary + } + field did type string { + indexing: attribute|index|summary + attribute : no-update + } + field categories type string { + indexing: attribute|index|summary + attribute : no-update + } + field pfrom type long { + indexing: attribute|summary + } + field pto type string { + indexing: summary + } + field data type string { + indexing: summary + } + field weight type float { + indexing { + field weight * 10 | summary; + } + } + field year type int { + indexing: attribute|summary + } + field sales type int { + indexing: attribute|summary + } + field surl type string { + indexing: summary + } + field pid type string { + indexing: index|summary + } + field ew type string { + indexing: index|summary + rank-type: about + } + field ed type string { + indexing: summary + } + field userrate type int { + indexing: summary + } + field numreview type int { + indexing: summary + } + field cbid type string { + indexing: attribute|index|summary + attribute : no-update + rank-type: about + } + field newestedition type int { + indexing: attribute|summary + } + field woty type int { + indexing: attribute|summary + } + field scid type string { + indexing: index|summary + rank-type: about + } + field w1 type float { + indexing { + field weight * 10 + field w1 | staticrank weight1 | summary; + } + } + field w2 type float { + indexing { + field w2 + field weight | staticrank weight2 | summary; + } + } + field w3 type float { + indexing { + field w3 + field weight | staticrank weight3 | summary; + } + } + field w4 type float { + indexing { + field w4 + field weight | staticrank weight4 | summary; + } + } + field sw1 type float { + indexing { + field weight * 10 + field w1 + field w2 | staticrank | summary; + } + } + field sw2 type float { + indexing { + field weight | staticrank sw2 | summary; + } + } + field sw3 type float { + indexing { + field weight | staticrank sw3 | summary; + } + } + field sw4 type float { + indexing { + field weight | staticrank sw4 | summary; + } + } + } + + field didinteger type int { + indexing { + field did | split_foreach " " { + attribute; + }; + } + attribute: multivalued + } + + rank-profile rp1 inherits default { + } + rank-profile rp2 inherits default { + } + rank-profile rp3 inherits default { + } + rank-profile rp4 inherits default { + } +} diff --git a/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..1f88a5e6477 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/MANIFEST.MF @@ -0,0 +1,7 @@ +Manifest-Version: 1.0 +Created-By: 1.6.0_20 (Apple Inc.) +Bundle-ManifestVersion: 2 +Bundle-Name: mybundle +Bundle-SymbolicName: mybundle +Bundle-Version: 0 + diff --git a/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/maven/com.yahoo.test/mybundle/pom.xml b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/maven/com.yahoo.test/mybundle/pom.xml new file mode 100644 index 00000000000..1d28f307824 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/pom-xml-warnings/META-INF/maven/com.yahoo.test/mybundle/pom.xml @@ -0,0 +1,15 @@ +<?xml version="1.0"?> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <groupId>com.yahoo.test</groupId> + <artifactId>mybundle</artifactId> + <packaging>container-plugin</packaging> + <version>1.0.0</version> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespa-http-client-extensions</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project>
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar Binary files differdeleted file mode 100644 index a395a52d17d..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF new file mode 100644 index 00000000000..21c58490c13 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/testjars/snapshot_bundle/META-INF/MANIFEST.MF @@ -0,0 +1,12 @@ +Manifest-Version: 1.0 +Archiver-Version: Plexus Archiver +Created-By: vespa container maven plugin +Built-By: tonyv +Build-Jdk: 1.6.0_26 +Bundle-Vendor: Yahoo! +Bundle-ClassPath: .,dependencies/jrt-5.1-SNAPSHOT.jar +Bundle-Version: 5.1.0.SNAPSHOT +Bundle-Name: container maven plugin test +Bundle-ManifestVersion: 2 +Bundle-SymbolicName: TestBundle + diff --git a/config-model/src/test/cfg/application/validation/testjars/test.jar b/config-model/src/test/cfg/application/validation/testjars/test.jar Binary files differdeleted file mode 100644 index 47fbd01f1ec..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/test.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar b/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar Binary files differdeleted file mode 100644 index 31266f1e8f2..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/wrong_classpath.jar +++ /dev/null diff --git a/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar b/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar Binary files differdeleted file mode 100644 index 47fbd01f1ec..00000000000 --- a/config-model/src/test/cfg/application/validation/testjars/wrong_export.jar +++ /dev/null diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 2447709f778..ba70b7493a2 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -2054,7 +2054,7 @@ public class ModelProvisioningTest { assertTrue("Initial servers are not joining", config.build().server().stream().noneMatch(ZookeeperServerConfig.Server::joining)); } { - VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(5), true, false, false, 0, Optional.of(model), new DeployState.Builder()); + VespaModel nextModel = tester.createModel(Zone.defaultZone(), servicesXml.apply(3), true, false, false, 0, Optional.of(model), new DeployState.Builder(), "node-1-3-10-04", "node-1-3-10-03"); ApplicationContainerCluster cluster = nextModel.getContainerClusters().get("zk"); ZookeeperServerConfig.Builder config = new ZookeeperServerConfig.Builder(); cluster.getContainers().forEach(c -> c.getConfig(config)); @@ -2067,6 +2067,14 @@ public class ModelProvisioningTest { 4, true), config.build().server().stream().collect(Collectors.toMap(ZookeeperServerConfig.Server::id, ZookeeperServerConfig.Server::joining))); + assertEquals("Retired nodes are retired", + Map.of(0, false, + 1, true, + 2, true, + 3, false, + 4, false), + config.build().server().stream().collect(Collectors.toMap(ZookeeperServerConfig.Server::id, + ZookeeperServerConfig.Server::retired))); } } @@ -2166,7 +2174,7 @@ public class ModelProvisioningTest { ProtonConfig cfg = getProtonConfig(model, cluster.getSearchNodes().get(0).getConfigId()); assertEquals(2000, cfg.flush().memory().maxtlssize()); // from config override assertEquals(1000, cfg.flush().memory().maxmemory()); // from explicit tuning - assertEquals((long) ((128 - reservedMemoryGb) * GB * 0.10), cfg.flush().memory().each().maxmemory()); // from default node flavor tuning + assertEquals((long) ((128 - reservedMemoryGb) * GB * 0.08), cfg.flush().memory().each().maxmemory()); // from default node flavor tuning } private static ProtonConfig getProtonConfig(VespaModel model, String configId) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java index c6a10c5530b..fda3e6c16c3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java @@ -104,8 +104,7 @@ public class ExportingTestCase extends AbstractExportingTestCase { @Test public void testRankExpression() throws IOException, ParseException { - assertCorrectDeriving("rankexpression", null, - new TestProperties().largeRankExpressionLimit(1024), new TestableDeployLogger()); + assertCorrectDeriving("rankexpression"); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index ebc8ceacc50..f4742be6b30 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -115,9 +115,9 @@ public class RankingExpressionsTestCase extends AbstractSchemaTestCase { @Test public void testLargeInheritedFunctions() throws IOException, ParseException { - ModelContext.Properties properties = new TestProperties().largeRankExpressionLimit(50); + ModelContext.Properties properties = new TestProperties(); RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - LargeRankExpressions largeExpressions = new LargeRankExpressions(new MockFileRegistry()); + LargeRankExpressions largeExpressions = new LargeRankExpressions(new MockFileRegistry(), 50); QueryProfileRegistry queryProfiles = new QueryProfileRegistry(); ImportedMlModels models = new ImportedMlModels(); Schema schema = createSearch("src/test/examples/largerankingexpressions", properties, rankProfileRegistry); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java new file mode 100644 index 00000000000..ef4353d02fb --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/BundleValidatorTest.java @@ -0,0 +1,111 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.application.provider.BaseDeployLogger; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarOutputStream; + +import static com.yahoo.yolean.Exceptions.uncheck; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class BundleValidatorTest { + @Rule + public TemporaryFolder tempDir = new TemporaryFolder(); + + @Test + public void basicBundleValidation() throws Exception { + // Valid jar file + JarFile ok = createTemporaryJarFile("ok"); + BundleValidator bundleValidator = new BundleValidator(); + bundleValidator.validateJarFile(new BaseDeployLogger(), ok); + + // No manifest + validateWithException("nomanifest", "Non-existing or invalid manifest in nomanifest.jar"); + } + + private void validateWithException(String jarName, String exceptionMessage) throws IOException { + try { + JarFile jarFile = createTemporaryJarFile(jarName); + BundleValidator bundleValidator = new BundleValidator(); + bundleValidator.validateJarFile(new BaseDeployLogger(), jarFile); + assert (false); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), exceptionMessage); + } + } + + @Test + public void require_that_deploying_snapshot_bundle_gives_warning() throws IOException { + final StringBuffer buffer = new StringBuffer(); + + DeployLogger logger = createDeployLogger(buffer); + JarFile jarFile = createTemporaryJarFile("snapshot_bundle"); + new BundleValidator().validateJarFile(logger, jarFile); + assertTrue(buffer.toString().contains("Deploying snapshot bundle")); + } + + @Test + public void outputs_deploy_warning_on_import_of_packages_from_deprecated_artifact() throws IOException { + final StringBuffer buffer = new StringBuffer(); + DeployLogger logger = createDeployLogger(buffer); + BundleValidator validator = new BundleValidator(); + JarFile jarFile = createTemporaryJarFile("import-warnings"); + validator.validateJarFile(logger, jarFile); + assertThat(buffer.toString()) + .contains("For JAR file 'import-warnings.jar': \n" + + "Manifest imports the following Java packages from 'org.json:json': [org.json]. \n" + + "The org.json library will no longer provided by jdisc runtime on Vespa 8. See https://docs.vespa.ai/en/vespa8-release-notes.html#container-runtime."); + } + + @Test + public void outputs_deploy_warning_on_deprecated_dependency() throws IOException { + StringBuffer buffer = new StringBuffer(); + DeployLogger logger = createDeployLogger(buffer); + BundleValidator validator = new BundleValidator(); + JarFile jarFile = createTemporaryJarFile("pom-xml-warnings"); + validator.validateJarFile(logger, jarFile); + assertThat(buffer.toString()) + .contains("The pom.xml of bundle 'pom-xml-warnings.jar' includes a dependency to the artifact " + + "'com.yahoo.vespa:vespa-http-client-extensions'. \n" + + "This artifact will be removed in Vespa 8. " + + "Programmatic use can be safely removed from system/staging tests. " + + "See internal Vespa 8 release notes for details.\n"); + } + + private JarFile createTemporaryJarFile(String testArtifact) throws IOException { + Path jarFile = tempDir.newFile(testArtifact + ".jar").toPath(); + Path artifactDirectory = Paths.get("src/test/cfg/application/validation/testjars/" + testArtifact); + try (JarOutputStream out = new JarOutputStream(Files.newOutputStream(jarFile))) { + Files.walk(artifactDirectory).forEach(path -> { + Path relativePath = artifactDirectory.relativize(path); + String zipName = relativePath.toString(); + uncheck(() -> { + if (Files.isDirectory(path)) { + out.putNextEntry(new JarEntry(zipName + "/")); + } else { + out.putNextEntry(new JarEntry(zipName)); + out.write(Files.readAllBytes(path)); + } + out.closeEntry(); + }); + }); + } + return new JarFile(jarFile.toFile()); + } + + private DeployLogger createDeployLogger(StringBuffer buffer) { + return (__, message) -> buffer.append(message).append('\n'); + } +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComponentValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComponentValidatorTest.java deleted file mode 100644 index a375621d391..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComponentValidatorTest.java +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import org.junit.Test; - -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.application.provider.BaseDeployLogger; - -import java.io.File; -import java.io.IOException; -import java.util.jar.JarFile; -import java.util.logging.Level; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class ComponentValidatorTest { - private static final String JARS_DIR = "src/test/cfg/application/validation/testjars/"; - - @Test - public void basicComponentValidation() throws Exception { - // Valid jar file - JarFile ok = new JarFile(new File(JARS_DIR + "ok.jar")); - ComponentValidator componentValidator = new ComponentValidator(ok); - componentValidator.validateAll(new BaseDeployLogger()); - - // No manifest - validateWithException("nomanifest.jar", "Non-existing or invalid manifest in " + JARS_DIR + "nomanifest.jar"); - } - - private void validateWithException(String jarName, String exceptionMessage) throws IOException { - try { - JarFile jarFile = new JarFile(JARS_DIR + jarName); - ComponentValidator componentValidator = new ComponentValidator(jarFile); - componentValidator.validateAll(new BaseDeployLogger()); - assert (false); - } catch (IllegalArgumentException e) { - assertEquals(e.getMessage(), exceptionMessage); - } - } - - @Test - public void require_that_deploying_snapshot_bundle_gives_warning() throws IOException { - final StringBuffer buffer = new StringBuffer(); - - DeployLogger logger = new DeployLogger() { - @Override - public void log(Level level, String message) { - buffer.append(message).append('\n'); - } - }; - - new ComponentValidator(new JarFile(JARS_DIR + "snapshot_bundle.jar")).validateAll(logger); - assertTrue(buffer.toString().contains("Deploying snapshot bundle")); - } -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index a27b6786800..c746db168ee 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -825,7 +825,7 @@ public class ContentBuilderTest extends DomBuilderTest { @Test public void feed_master_task_limit_is_controlled_by_feature_flag() { - assertEquals(0, resolveFeedMasterTaskLimitConfigWithFeatureFlag(null)); + assertEquals(1000, resolveFeedMasterTaskLimitConfigWithFeatureFlag(null)); assertEquals(2000, resolveFeedMasterTaskLimitConfigWithFeatureFlag(2000)); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java index 7f49efa8770..4ca85a19c35 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/ConfigserverClusterTest.java @@ -47,16 +47,17 @@ public class ConfigserverClusterTest { } @Test - public void zookeeperConfig_only_config_servers_set() { + public void zookeeperConfig_only_config_servers_set_hosted() { TestOptions testOptions = createTestOptions(Arrays.asList("cfg1", "localhost", "cfg3"), Collections.emptyList()); ZookeeperServerConfig config = getConfig(ZookeeperServerConfig.class, testOptions); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 0, 1, 2); assertEquals(1, config.myid()); + assertEquals("gz", config.snapshotMethod()); } @Test - public void zookeeperConfig_with_config_servers_and_zk_ids() { + public void zookeeperConfig_with_config_servers_and_zk_ids_hosted() { TestOptions testOptions = createTestOptions(Arrays.asList("cfg1", "localhost", "cfg3"), Arrays.asList(4, 2, 3)); ZookeeperServerConfig config = getConfig(ZookeeperServerConfig.class, testOptions); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); @@ -72,6 +73,7 @@ public class ConfigserverClusterTest { assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::hostname, "cfg1", "localhost", "cfg3"); assertZookeeperServerProperty(config.server(), ZookeeperServerConfig.Server::id, 4, 2, 3); assertEquals(2, config.myid()); + assertEquals("", config.snapshotMethod()); } @Test(expected = IllegalArgumentException.class) @@ -150,7 +152,8 @@ public class ConfigserverClusterTest { .useVespaVersionInRequest(true) .hostedVespa(hostedVespa) .environment("test") - .region("bar"); + .region("bar") + .zooKeeperSnapshotMethod(hostedVespa ? "gz" : ""); Optional.of(configServerHostnames) .filter(hostnames -> !hostnames.isEmpty()) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index 9352dac85a4..fc7f8674149 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.container.configserver; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; +import java.util.Objects; import java.util.Optional; /** @@ -17,6 +18,7 @@ public class TestOptions implements CloudConfigOptions { private Optional<String> region = Optional.empty(); private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); + private String zooKeeperSnapshotMethod = ""; @Override public Optional<Integer> rpcPort() { @@ -106,6 +108,9 @@ public class TestOptions implements CloudConfigOptions { return Optional.empty(); } + @Override + public String zooKeeperSnapshotMethod() { return zooKeeperSnapshotMethod; } + public TestOptions configServers(ConfigServer[] configServers) { this.configServers = configServers; return this; @@ -130,4 +135,11 @@ public class TestOptions implements CloudConfigOptions { this.hostedVespa = Optional.of(hostedVespa); return this; } + + public TestOptions zooKeeperSnapshotMethod(String snapshotMethod) { + Objects.requireNonNull(snapshotMethod); + this.zooKeeperSnapshotMethod = snapshotMethod; + return this; + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/SemanticRulesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/SemanticRulesTest.java index 68690bd83cd..8d83ec4cc5f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/SemanticRulesTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/SemanticRulesTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.container.search; import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleImporter; import com.yahoo.prelude.semantics.SemanticRulesConfig; @@ -39,7 +40,7 @@ public class SemanticRulesTest { } private static Map<String, RuleBase> toMap(SemanticRulesConfig config) throws ParseException, IOException { - RuleImporter ruleImporter = new RuleImporter(config); + RuleImporter ruleImporter = new RuleImporter(config, new SimpleLinguistics()); Map<String, RuleBase> ruleBaseMap = new HashMap<>(); for (SemanticRulesConfig.Rulebase ruleBaseConfig : config.rulebase()) { RuleBase ruleBase = ruleImporter.importConfig(ruleBaseConfig); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index 96fbce03e88..814b2f94e44 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -181,7 +181,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { "-XX:+ParallelGCThreads=8 foo bar"); fail(); } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Invalid JVM GC options in services.xml: bar,foo")); + assertTrue(e.getMessage().startsWith("Invalid or misplaced JVM GC options in services.xml: bar,foo")); } } @@ -196,15 +196,19 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { List<String> strings = Arrays.asList(invalidOptions.clone()); // Verify that nothing is logged if there are no invalid options if (strings.isEmpty()) { - assertEquals(0, logger.msgs.size()); + assertEquals(logger.msgs.size() > 0 ? logger.msgs.get(0).getSecond() : "", 0, logger.msgs.size()); return; } - Collections.sort(strings); + assertTrue("Expected 1 or more log messages for invalid JM options, got none", logger.msgs.size() > 0); Pair<Level, String> firstOption = logger.msgs.get(0); assertEquals(Level.WARNING, firstOption.getFirst()); - assertEquals("Invalid JVM " + (optionName.equals("gc-options") ? "GC " : "") + - "options in services.xml: " + String.join(",", strings), firstOption.getSecond()); + + Collections.sort(strings); + assertEquals("Invalid or misplaced JVM" + (optionName.equals("gc-options") ? " GC" : "") + + " options in services.xml: " + String.join(",", strings) + "." + + " See https://docs.vespa.ai/en/reference/services-container.html#jvm" + , firstOption.getSecond()); } private void buildModelWithJvmOptions(boolean isHosted, TestLogger logger, String optionName, String override) throws IOException, SAXException { @@ -238,6 +242,11 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { "$(touch /tmp/hello-from-gc-options)", "$(touch", "/tmp/hello-from-gc-options)"); + verifyLoggingOfJvmOptions(true, + "options", + "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005", + "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"); + verifyLoggingOfJvmOptions(false, "options", "$(touch /tmp/hello-from-gc-options)", @@ -247,6 +256,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { verifyLoggingOfJvmOptions(true, "options", "-Xms2G"); verifyLoggingOfJvmOptions(true, "options", "-verbose:gc"); verifyLoggingOfJvmOptions(true, "options", "-Djava.library.path=/opt/vespa/lib64:/home/y/lib64"); + verifyLoggingOfJvmOptions(true, "options", "-XX:-OmitStackTraceInFastThrow"); verifyLoggingOfJvmOptions(false, "options", "-Xms2G"); } @@ -259,7 +269,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { "-Xms2G foo bar"); fail(); } catch (IllegalArgumentException e) { - assertTrue(e.getMessage().contains("Invalid JVM options in services.xml: bar,foo")); + assertTrue(e.getMessage().contains("Invalid or misplaced JVM options in services.xml: bar,foo")); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 191f7f5652a..87a962339e9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -45,7 +45,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalDouble; import java.util.OptionalInt; import static org.junit.Assert.assertEquals; @@ -1046,11 +1045,9 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(7, resolveMaxCompactBuffers(OptionalInt.of(7))); } - private long resolveMaxTLSSize(OptionalDouble tlsSizeFraction, Optional<Flavor> flavor) throws Exception { + private long resolveMaxTLSSize(Optional<Flavor> flavor) throws Exception { TestProperties testProperties = new TestProperties(); - if (tlsSizeFraction.isPresent()) { - testProperties.tlsSizeFraction(tlsSizeFraction.getAsDouble()); - } + ContentCluster cc = createOneNodeCluster(testProperties, flavor); ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); cc.getSearch().getSearchNodes().get(0).getConfig(protonBuilder); @@ -1058,13 +1055,10 @@ public class ContentClusterTest extends ContentBaseTest { return protonConfig.flush().memory().maxtlssize(); } @Test - public void default_max_tls_size_controlled_by_properties() throws Exception { + public void verifyt_max_tls_size() throws Exception { var flavor = new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder().name("test").minDiskAvailableGb(100))); - assertEquals(21474836480L, resolveMaxTLSSize(OptionalDouble.empty(), Optional.empty())); - assertEquals(21474836480L, resolveMaxTLSSize(OptionalDouble.of(0.02), Optional.empty())); - assertEquals(7516192768L, resolveMaxTLSSize(OptionalDouble.empty(), Optional.of(flavor))); - assertEquals(2147483648L, resolveMaxTLSSize(OptionalDouble.of(0.02), Optional.of(flavor))); - assertEquals(3221225472L, resolveMaxTLSSize(OptionalDouble.of(0.03), Optional.of(flavor))); + assertEquals(21474836480L, resolveMaxTLSSize(Optional.empty())); + assertEquals(2147483648L, resolveMaxTLSSize(Optional.of(flavor))); } void assertZookeeperServerImplementation(String expectedClassName, @@ -1127,8 +1121,8 @@ public class ContentClusterTest extends ContentBaseTest { @Test public void distributor_merge_busy_wait_controlled_by_properties() throws Exception { - assertEquals(10, resolveDistributorMergeBusyWaitConfig(Optional.empty())); - assertEquals(1, resolveDistributorMergeBusyWaitConfig(Optional.of(1))); + assertEquals(1, resolveDistributorMergeBusyWaitConfig(Optional.empty())); + assertEquals(5, resolveDistributorMergeBusyWaitConfig(Optional.of(5))); } private int resolveDistributorMergeBusyWaitConfig(Optional<Integer> mergeBusyWait) throws Exception { @@ -1144,14 +1138,14 @@ public class ContentClusterTest extends ContentBaseTest { @Test public void distributor_enhanced_maintenance_scheduling_controlled_by_properties() throws Exception { - assertFalse(resolveDistributorEnhancedSchedulingConfig(false)); - assertTrue(resolveDistributorEnhancedSchedulingConfig(true)); + assertFalse(resolveDistributorEnhancedSchedulingConfig(Optional.of(false))); + assertTrue(resolveDistributorEnhancedSchedulingConfig(Optional.empty())); } - private boolean resolveDistributorEnhancedSchedulingConfig(boolean enhancedScheduling) throws Exception { + private boolean resolveDistributorEnhancedSchedulingConfig(Optional<Boolean> enhancedScheduling) throws Exception { var props = new TestProperties(); - if (enhancedScheduling) { - props.distributorEnhancedMaintenanceScheduling(enhancedScheduling); + if (enhancedScheduling.isPresent()) { + props.distributorEnhancedMaintenanceScheduling(enhancedScheduling.get()); } var cluster = createOneNodeCluster(props); var builder = new StorDistributormanagerConfig.Builder(); @@ -1161,14 +1155,14 @@ public class ContentClusterTest extends ContentBaseTest { @Test public void unordered_merge_chaining_config_controlled_by_properties() throws Exception { - assertFalse(resolveUnorderedMergeChainingConfig(false)); - assertTrue(resolveUnorderedMergeChainingConfig(true)); + assertFalse(resolveUnorderedMergeChainingConfig(Optional.of(false))); + assertTrue(resolveUnorderedMergeChainingConfig(Optional.empty())); } - private boolean resolveUnorderedMergeChainingConfig(boolean unorderedMergeChaining) throws Exception { + private boolean resolveUnorderedMergeChainingConfig(Optional<Boolean> unorderedMergeChaining) throws Exception { var props = new TestProperties(); - if (unorderedMergeChaining) { - props.setUnorderedMergeChaining(true); + if (unorderedMergeChaining.isPresent()) { + props.setUnorderedMergeChaining(unorderedMergeChaining.get()); } var cluster = createOneNodeCluster(props); var builder = new StorDistributormanagerConfig.Builder(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java index d6020a96818..68e722f45d3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java @@ -33,7 +33,7 @@ import static org.junit.Assert.assertTrue; */ public class ContentSchemaClusterTest { - private static double EPSILON = 0.000001; + private static final double EPSILON = 0.000001; private static ContentCluster createClusterWithOneDocumentType() throws Exception { return createCluster(new ContentClusterBuilder().getXml()); @@ -261,27 +261,16 @@ public class ContentSchemaClusterTest { } @Test - public void verifyControlOfDocStoreCompression() throws Exception { + public void verifyDefaultDocStoreCompression() throws Exception { ProtonConfig cfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml())); - assertEquals(9, cfg.summary().log().chunk().compression().level()); - assertEquals(9, cfg.summary().log().compact().compression().level()); - - cfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml(), - new DeployState.Builder().properties(new TestProperties().docstoreCompressionLevel(3)))); assertEquals(3, cfg.summary().log().chunk().compression().level()); assertEquals(3, cfg.summary().log().compact().compression().level()); } @Test - public void verifyControlOfDiskBloatFactor() throws Exception { + public void verifyDefaultDiskBloatFactor() throws Exception { var defaultCfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml())); - assertEquals(0.2, defaultCfg.flush().memory().diskbloatfactor(), EPSILON); - assertEquals(0.2, defaultCfg.flush().memory().each().diskbloatfactor(), EPSILON); - - var controlledCfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml(), - new DeployState.Builder().properties(new TestProperties().diskBloatFactor(0.31)) - )); - assertEquals(0.31, controlledCfg.flush().memory().diskbloatfactor(), EPSILON); - assertEquals(0.31, controlledCfg.flush().memory().each().diskbloatfactor(), EPSILON); + assertEquals(0.25, defaultCfg.flush().memory().diskbloatfactor(), EPSILON); + assertEquals(0.25, defaultCfg.flush().memory().each().diskbloatfactor(), EPSILON); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 739f8b7fff2..cf877d3bf88 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -117,8 +117,8 @@ public class StorageClusterTest { StorServerConfig config = new StorServerConfig(builder); assertEquals(16, config.max_merges_per_node()); - assertEquals(1024, config.max_merge_queue_size()); - assertFalse(config.disable_queue_limits_for_chained_merges()); + assertEquals(100, config.max_merge_queue_size()); + assertTrue(config.disable_queue_limits_for_chained_merges()); } @Test @@ -171,10 +171,10 @@ public class StorageClusterTest { @Test public void async_apply_bucket_diff_can_be_controlled_by_feature_flag() { var config = filestorConfigFromProperties(new TestProperties()); - assertFalse(config.async_apply_bucket_diff()); - - config = filestorConfigFromProperties(new TestProperties().setAsyncApplyBucketDiff(true)); assertTrue(config.async_apply_bucket_diff()); + + config = filestorConfigFromProperties(new TestProperties().setAsyncApplyBucketDiff(false)); + assertFalse(config.async_apply_bucket_diff()); } @Test @@ -306,6 +306,25 @@ public class StorageClusterTest { } @Test + public void persistence_async_throttle_config_defaults_to_unlimited() { + var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + } + + @Test + public void persistence_async_throttle_config_is_derived_from_flag() { + var config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("UNLIMITED"))); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + + config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("DYNAMIC"))); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type()); + + // Invalid enum values fall back to the default + config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("BANANAS"))); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + } + + @Test public void integrity_checker_explicitly_disabled_when_not_running_with_vds_provider() { StorIntegritycheckerConfig.Builder builder = new StorIntegritycheckerConfig.Builder(); parse(cluster("bees", "")).getConfig(builder); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index 9a6a7458766..5571ead11ce 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -23,6 +23,7 @@ public class NodeResourcesTuningTest { private static final double delta = 0.00001; private static final double combinedFactor = 1 - 18.0/100; + private static final double DEFAULT_MEMORY_GAIN = 0.08; @Test public void require_that_hwinfo_disk_size_is_set() { @@ -41,7 +42,7 @@ public class NodeResourcesTuningTest { assertEquals(0.5, reservedMemoryGb, delta); } - private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, double gb, int redundancy, int searchableCopies) { + private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, double gb) { ProtonConfig.Builder builder = new ProtonConfig.Builder(); for (Pair<String, String> sdMode : sdAndMode) { builder.documentdb.add(new ProtonConfig.Documentdb.Builder() @@ -52,8 +53,8 @@ public class NodeResourcesTuningTest { return configFromMemorySetting(gb, builder); } - private void verify_that_initial_numdocs_is_dependent_of_mode(int redundancy, int searchablecopies) { - ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24 + reservedMemoryGb, redundancy, searchablecopies); + private void verify_that_initial_numdocs_is_dependent_of_mode() { + ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24 + reservedMemoryGb); assertEquals(3, cfg.documentdb().size()); assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs()); assertEquals("a", cfg.documentdb(0).inputdoctypename()); @@ -65,10 +66,8 @@ public class NodeResourcesTuningTest { @Test public void require_that_initial_numdocs_is_dependent_of_mode_and_searchablecopies() { - verify_that_initial_numdocs_is_dependent_of_mode(2,0); - verify_that_initial_numdocs_is_dependent_of_mode(1,1); - verify_that_initial_numdocs_is_dependent_of_mode(3, 2); - verify_that_initial_numdocs_is_dependent_of_mode(3, 3); + verify_that_initial_numdocs_is_dependent_of_mode(); + } @Test @@ -125,21 +124,19 @@ public class NodeResourcesTuningTest { @Test public void require_that_flush_strategy_memory_limits_are_set_based_on_available_memory() { - assertFlushStrategyMemory((long)(4 * GB * 0.10), 4); - assertFlushStrategyMemory((long)(8 * GB * 0.10), 8); - assertFlushStrategyMemory((long)(24 * GB * 0.10), 24); - assertFlushStrategyMemory((long)(64 * GB * 0.10), 64); + assertFlushStrategyMemory((long)(4 * GB * DEFAULT_MEMORY_GAIN), 4); + assertFlushStrategyMemory((long)(8 * GB * DEFAULT_MEMORY_GAIN), 8); + assertFlushStrategyMemory((long)(24 * GB * DEFAULT_MEMORY_GAIN), 24); + assertFlushStrategyMemory((long)(64 * GB * DEFAULT_MEMORY_GAIN), 64); } @Test public void require_that_flush_strategy_tls_size_is_set_based_on_available_disk() { - assertFlushStrategyTlsSize(2 * GB, 10, 0.05); - assertFlushStrategyTlsSize(7 * GB, 100, 0.07); - assertFlushStrategyTlsSize(5 * GB, 100, 0.05); - assertFlushStrategyTlsSize(35 * GB, 500, 0.07); - assertFlushStrategyTlsSize(84 * GB, 1200, 0.07); - assertFlushStrategyTlsSize(100 * GB, 1720, 0.07); - assertFlushStrategyTlsSize(100 * GB, 24000, 0.07); + assertFlushStrategyTlsSize(2 * GB, 10); + assertFlushStrategyTlsSize(2 * GB, 100); + assertFlushStrategyTlsSize(10 * GB, 500); + assertFlushStrategyTlsSize(24 * GB, 1200); + assertFlushStrategyTlsSize(100 * GB, 24000); } @Test @@ -156,14 +153,14 @@ public class NodeResourcesTuningTest { @Test public void require_that_summary_cache_max_bytes_is_set_based_on_memory() { - assertEquals(1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, 0).summary().cache().maxbytes()); - assertEquals(256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, 0).summary().cache().maxbytes()); + assertEquals(1*GB / 25, configFromMemorySetting(1 + reservedMemoryGb, 0).summary().cache().maxbytes()); + assertEquals(256*GB / 25, configFromMemorySetting(256 + reservedMemoryGb, 0).summary().cache().maxbytes()); } @Test public void require_that_summary_cache_memory_is_reduced_with_combined_cluster() { - assertEquals(combinedFactor * 1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster*0.01).summary().cache().maxbytes(), 1000); - assertEquals(combinedFactor * 256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster*0.01).summary().cache().maxbytes(), 1000); + assertEquals(combinedFactor * 1*GB / 25, configFromMemorySetting(1 + reservedMemoryGb, ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster*0.01).summary().cache().maxbytes(), 1000); + assertEquals(combinedFactor * 256*GB / 25, configFromMemorySetting(256 + reservedMemoryGb, ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster*0.01).summary().cache().maxbytes(), 1000); } @Test @@ -180,8 +177,8 @@ public class NodeResourcesTuningTest { assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, 0).flush().memory().each().maxmemory()); } - private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb, double tlsSizeFraction) { - assertEquals(expTlsSizeBytes, configFromDiskSetting(diskGb, tlsSizeFraction).flush().memory().maxtlssize()); + private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb) { + assertEquals(expTlsSizeBytes, configFromDiskSetting(diskGb).flush().memory().maxtlssize()); } private static void assertSummaryReadIo(ProtonConfig.Summary.Read.Io.Enum expValue, boolean fastDisk) { @@ -196,23 +193,16 @@ public class NodeResourcesTuningTest { assertEquals(sharedDisk, configFromEnvironmentType(docker).hwinfo().disk().shared()); } - private static void assertWriteFilter(double expMemoryLimit, int memoryGb) { - assertEquals(expMemoryLimit, configFromMemorySetting(memoryGb, 0).writefilter().memorylimit(), delta); - } - private static ProtonConfig configFromDiskSetting(boolean fastDisk) { return getConfig(new FlavorsConfig.Flavor.Builder().fastDisk(fastDisk)); } private static ProtonConfig configFromDiskSetting(int diskGb) { - return configFromDiskSetting(diskGb, 0.07); - } - private static ProtonConfig configFromDiskSetting(int diskGb, double tlsSizeFraction) { - return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), 0, tlsSizeFraction); + return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), 0); } private static ProtonConfig configFromMemorySetting(double memoryGb, double fractionOfMemoryReserved) { - return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), fractionOfMemoryReserved, 0.07); + return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), fractionOfMemoryReserved); } private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder) { @@ -238,26 +228,26 @@ public class NodeResourcesTuningTest { return getConfig(flavorBuilder, new ProtonConfig.Builder()); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, double fractionOfMemoryReserved, double tlsSizeFraction) { - return getConfig(flavorBuilder, new ProtonConfig.Builder(), fractionOfMemoryReserved, tlsSizeFraction); + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, double fractionOfMemoryReserved) { + return getConfig(flavorBuilder, new ProtonConfig.Builder(), fractionOfMemoryReserved); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) { return getConfig(flavorBuilder, protonBuilder,1); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, double fractionOfMemoryReserved, double tlsSizeFraction) { - return getConfig(flavorBuilder, protonBuilder, 1, fractionOfMemoryReserved, tlsSizeFraction); + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, double fractionOfMemoryReserved) { + return getConfig(flavorBuilder, protonBuilder, 1, fractionOfMemoryReserved); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int numThreadsPerSearch) { - return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0, 0.07); + return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, - int numThreadsPerSearch, double fractionOfMemoryReserved, double tlsSizeFraction) { + int numThreadsPerSearch, double fractionOfMemoryReserved) { flavorBuilder.name("my_flavor"); - NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, fractionOfMemoryReserved, tlsSizeFraction); + NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, fractionOfMemoryReserved); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java index d000f83d6cd..226045f4d8a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java @@ -51,8 +51,7 @@ public class SchemaNodeTest { private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted) { return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, - Optional.empty(), Optional.empty(), isHosted, 0.0, - root.getDeployState().featureFlags().tlsSizeFraction()); + Optional.empty(), Optional.empty(), isHosted, 0.0); } private static SearchNode createSearchNode(MockRoot root) { diff --git a/config-provisioning/src/main/resources/configdefinitions/config.provisioning.node-repository.def b/config-provisioning/src/main/resources/configdefinitions/config.provisioning.node-repository.def index 6181efc7184..b054f434322 100644 --- a/config-provisioning/src/main/resources/configdefinitions/config.provisioning.node-repository.def +++ b/config-provisioning/src/main/resources/configdefinitions/config.provisioning.node-repository.def @@ -11,4 +11,4 @@ tenantContainerImage string default="" useCuratorClientCache bool default=false # The number of Node objects to cache in-memory. -nodeCacheSize long default=2000 +nodeCacheSize long default=3000 diff --git a/config-proxy/pom.xml b/config-proxy/pom.xml index e79aec952db..a855d84de71 100644 --- a/config-proxy/pom.xml +++ b/config-proxy/pom.xml @@ -85,12 +85,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index d8459b175a7..a7f6a2a97a7 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -106,7 +106,7 @@ export LD_LIBRARY_PATH="$VESPA_HOME/lib64" case $1 in start) - nohup sbin/vespa-retention-enforcer > ${LOGDIR}/vre-start.log 2>&1 </dev/null & + nohup nice sbin/vespa-retention-enforcer > ${LOGDIR}/vre-start.log 2>&1 </dev/null & configsources=`bin/vespa-print-default configservers_rpc` userargs=$VESPA_CONFIGPROXY_JVMARGS jvmopts="-Xms32M -Xmx128M -XX:CompressedClassSpaceSize=32m -XX:MaxDirectMemorySize=32m -XX:ThreadStackSize=256 -XX:MaxJavaStackTraceDepth=1000 -XX:-OmitStackTraceInFastThrow" diff --git a/configdefinitions/pom.xml b/configdefinitions/pom.xml index 78aa8b59222..a90d656e1d6 100644 --- a/configdefinitions/pom.xml +++ b/configdefinitions/pom.xml @@ -30,12 +30,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> </plugin> <plugin> <groupId>com.yahoo.vespa</groupId> diff --git a/configdefinitions/src/main/java/com/yahoo/vespa/configdefinition/package-info.java b/configdefinitions/src/main/java/com/yahoo/vespa/configdefinition/package-info.java index 06c8e57c92f..f00b5bdf122 100644 --- a/configdefinitions/src/main/java/com/yahoo/vespa/configdefinition/package-info.java +++ b/configdefinitions/src/main/java/com/yahoo/vespa/configdefinition/package-info.java @@ -2,5 +2,4 @@ @ExportPackage package com.yahoo.vespa.configdefinition; -import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configdefinitions/src/main/java/com/yahoo/vespa/orchestrator/config/package-info.java b/configdefinitions/src/main/java/com/yahoo/vespa/orchestrator/config/package-info.java index af2e374fc9c..3d2f69bb036 100644 --- a/configdefinitions/src/main/java/com/yahoo/vespa/orchestrator/config/package-info.java +++ b/configdefinitions/src/main/java/com/yahoo/vespa/orchestrator/config/package-info.java @@ -2,5 +2,4 @@ @ExportPackage package com.yahoo.vespa.orchestrator.config; -import com.yahoo.api.annotations.PublicApi; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/configdefinitions/src/vespa/stor-filestor.def b/configdefinitions/src/vespa/stor-filestor.def index 66700eff3e6..1aaa7f71f8e 100644 --- a/configdefinitions/src/vespa/stor-filestor.def +++ b/configdefinitions/src/vespa/stor-filestor.def @@ -45,7 +45,7 @@ bucket_merge_chunk_size int default=33554432 restart ## If set, portions of apply bucket diff handling will be performed asynchronously ## with persistence thread not waiting for local writes to complete. -async_apply_bucket_diff bool default=false +async_apply_bucket_diff bool default=true ## When merging, it is possible to send more metadata than needed in order to ## let local nodes in merge decide which entries fits best to add this time @@ -75,3 +75,22 @@ use_async_message_handling_on_schedule bool default=false restart ## the entire resource usage sample is immediately reported to the cluster controller (via host info). ## This config can be live updated (doesn't require restart). resource_usage_reporter_noise_level double default=0.001 + +## Specify throttling used for async persistence operations. This throttling takes place +## before operations are dispatched to Proton and serves as a limiter for how many +## operations may be in flight in Proton's internal queues. +## +## - UNLIMITED is, as it says on the tin, unlimited. Offers no actual throttling, but +## has near zero overhead and never blocks. +## - DYNAMIC uses DynamicThrottlePolicy under the hood and will block if the window +## is full (if a blocking throttler API call is invoked). +## +async_operation_throttler_type enum { UNLIMITED, DYNAMIC } default=UNLIMITED restart + +## Specifies the extent the throttling window is increased by when the async throttle +## policy has decided that more concurrent operations are desirable. Also affects the +## _minimum_ size of the throttling window; its size is implicitly set to max(this config +## value, number of threads). +## +## Only applies if async_operation_throttler_type == DYNAMIC. +async_operation_dynamic_throttling_window_increment int default=20 restart diff --git a/configdefinitions/src/vespa/zookeeper-server.def b/configdefinitions/src/vespa/zookeeper-server.def index cf923d58d7d..d80ccc4d042 100644 --- a/configdefinitions/src/vespa/zookeeper-server.def +++ b/configdefinitions/src/vespa/zookeeper-server.def @@ -32,10 +32,13 @@ juteMaxBuffer int default=52428800 myid int restart server[].id int server[].hostname string +server[].clientPort int default=2181 server[].quorumPort int default=2182 server[].electionPort int default=2183 # Whether this server is joining an existing cluster server[].joining bool default=false +# Whether this server is retired, and about to be removed +server[].retired bool default=false # Needed when upgrading from ZooKeeper 3.4 to 3.5, see https://issues.apache.org/jira/browse/ZOOKEEPER-3056, # and in general where there is a zookeeper ensemble running that has had few transactions. @@ -51,3 +54,4 @@ tlsForClientServerCommunication enum { OFF, PORT_UNIFICATION, TLS_WITH_PORT_UNIF jksKeyStoreFile string default="conf/zookeeper/zookeeper.jks" dynamicReconfiguration bool default=false +snapshotMethod string default="" diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 02ca4ce14c4..80194337daa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -111,6 +111,8 @@ import java.util.stream.Collectors; import static com.yahoo.config.model.api.container.ContainerServiceType.CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.LOGSERVER_CONTAINER; +import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse; +import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.fileReferenceExistsOnDisk; import static com.yahoo.vespa.config.server.filedistribution.FileDistributionUtil.getFileReferencesOnDisk; import static com.yahoo.vespa.config.server.tenant.TenantRepository.HOSTED_VESPA_TENANT; @@ -737,16 +739,22 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- Convergence ---------------------------------------------------------------- - public HttpResponse checkServiceForConfigConvergence(ApplicationId applicationId, String hostAndPort, URI uri, - Duration timeout, Optional<Version> vespaVersion) { - return convergeChecker.getServiceConfigGenerationResponse(getApplication(applicationId, vespaVersion), hostAndPort, uri, timeout); + public ServiceResponse checkServiceForConfigConvergence(ApplicationId applicationId, + String hostAndPort, + Duration timeout, + Optional<Version> vespaVersion) { + return convergeChecker.getServiceConfigGeneration(getApplication(applicationId, vespaVersion), hostAndPort, timeout); } - public HttpResponse servicesToCheckForConfigConvergence(ApplicationId applicationId, URI uri, - Duration timeoutPerService, Optional<Version> vespaVersion) { - return convergeChecker.getServiceConfigGenerationsResponse(getApplication(applicationId, vespaVersion), uri, timeoutPerService); + public ServiceListResponse servicesToCheckForConfigConvergence(ApplicationId applicationId, + URI uri, + Duration timeoutPerService, + Optional<Version> vespaVersion) { + return convergeChecker.getServiceConfigGenerations(getApplication(applicationId, vespaVersion), uri, timeoutPerService); } + public ConfigConvergenceChecker configConvergenceChecker() { return convergeChecker; } + // ---------------- Logs ---------------------------------------------------------------- public HttpResponse getLogs(ApplicationId applicationId, Optional<String> hostname, String apiParams) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index 24744a1b3b2..ad14cf4aab6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -10,8 +10,6 @@ import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.slime.Cursor; -import com.yahoo.vespa.config.server.http.JSONResponse; import org.apache.hc.client5.http.async.methods.SimpleHttpRequest; import org.apache.hc.client5.http.async.methods.SimpleHttpResponse; import org.apache.hc.client5.http.async.methods.SimpleRequestBuilder; @@ -92,27 +90,26 @@ public class ConfigConvergenceChecker extends AbstractComponent { } /** Check all services in given application. Returns the minimum current generation of all services */ - public JSONResponse getServiceConfigGenerationsResponse(Application application, URI requestUrl, Duration timeoutPerService) { + public ServiceListResponse getServiceConfigGenerations(Application application, URI uri, Duration timeoutPerService) { Map<ServiceInfo, Long> currentGenerations = getServiceConfigGenerations(application, timeoutPerService); long currentGeneration = currentGenerations.values().stream().mapToLong(Long::longValue).min().orElse(-1); - return new ServiceListResponse(200, currentGenerations, requestUrl, application.getApplicationGeneration(), - currentGeneration); + return new ServiceListResponse(currentGenerations, uri, application.getApplicationGeneration(), currentGeneration); } /** Check service identified by host and port in given application */ - public JSONResponse getServiceConfigGenerationResponse(Application application, String hostAndPortToCheck, URI requestUrl, Duration timeout) { + public ServiceResponse getServiceConfigGeneration(Application application, String hostAndPortToCheck, Duration timeout) { Long wantedGeneration = application.getApplicationGeneration(); try (CloseableHttpAsyncClient client = createHttpClient()) { client.start(); if ( ! hostInApplication(application, hostAndPortToCheck)) - return ServiceResponse.createHostNotFoundInAppResponse(requestUrl, hostAndPortToCheck, wantedGeneration); + return new ServiceResponse(ServiceResponse.Status.hostNotFound, wantedGeneration); long currentGeneration = getServiceGeneration(client, URI.create("http://" + hostAndPortToCheck), timeout).get(); boolean converged = currentGeneration >= wantedGeneration; - return ServiceResponse.createOkResponse(requestUrl, hostAndPortToCheck, wantedGeneration, currentGeneration, converged); + return new ServiceResponse(ServiceResponse.Status.ok, wantedGeneration, currentGeneration, converged); } catch (InterruptedException | ExecutionException | CancellationException e) { // e.g. if we cannot connect to the service to find generation - return ServiceResponse.createNotFoundResponse(requestUrl, hostAndPortToCheck, wantedGeneration, e.getMessage()); + return new ServiceResponse(ServiceResponse.Status.notFound, wantedGeneration, e.getMessage()); } catch (Exception e) { - return ServiceResponse.createErrorResponse(requestUrl, hostAndPortToCheck, wantedGeneration, e.getMessage()); + return new ServiceResponse(ServiceResponse.Status.error, wantedGeneration, e.getMessage()); } } @@ -192,7 +189,7 @@ public class ConfigConvergenceChecker extends AbstractComponent { return false; } - private static Optional<Integer> getStatePort(ServiceInfo service) { + public static Optional<Integer> getStatePort(ServiceInfo service) { return service.getPorts().stream() .filter(port -> port.getTags().contains("state")) .map(PortInfo::getPort) @@ -249,63 +246,70 @@ public class ConfigConvergenceChecker extends AbstractComponent { .build(); } - private static class ServiceListResponse extends JSONResponse { - - // Pre-condition: servicesToCheck has a state port - private ServiceListResponse(int status, Map<ServiceInfo, Long> servicesToCheck, URI uri, long wantedGeneration, - long currentGeneration) { - super(status); - Cursor serviceArray = object.setArray("services"); - servicesToCheck.forEach((service, generation) -> { - Cursor serviceObject = serviceArray.addObject(); - String hostName = service.getHostName(); - int statePort = getStatePort(service).get(); - serviceObject.setString("host", hostName); - serviceObject.setLong("port", statePort); - serviceObject.setString("type", service.getServiceType()); - serviceObject.setString("url", uri.toString() + "/" + hostName + ":" + statePort); - serviceObject.setLong("currentGeneration", generation); - }); - object.setString("url", uri.toString()); - object.setLong("currentGeneration", currentGeneration); - object.setLong("wantedGeneration", wantedGeneration); - object.setBool("converged", currentGeneration >= wantedGeneration); + public static class ServiceResponse { + + public enum Status { ok, notFound, hostNotFound, error } + + public final Status status; + public final Long wantedGeneration; + public final Long currentGeneration; + public final boolean converged; + public final Optional<String> errorMessage; + + public ServiceResponse(Status status, Long wantedGeneration) { + this(status, wantedGeneration, 0L); } - } - private static class ServiceResponse extends JSONResponse { + public ServiceResponse(Status status, Long wantedGeneration, Long currentGeneration) { + this(status, wantedGeneration, currentGeneration, false); + } - private ServiceResponse(int status, URI uri, String hostname, Long wantedGeneration) { - super(status); - object.setString("url", uri.toString()); - object.setString("host", hostname); - object.setLong("wantedGeneration", wantedGeneration); + public ServiceResponse(Status status, Long wantedGeneration, Long currentGeneration, boolean converged) { + this(status, wantedGeneration, currentGeneration, converged, Optional.empty()); } - static ServiceResponse createOkResponse(URI uri, String hostname, Long wantedGeneration, Long currentGeneration, boolean converged) { - ServiceResponse serviceResponse = new ServiceResponse(200, uri, hostname, wantedGeneration); - serviceResponse.object.setBool("converged", converged); - serviceResponse.object.setLong("currentGeneration", currentGeneration); - return serviceResponse; + public ServiceResponse(Status status, Long wantedGeneration, String errorMessage) { + this(status, wantedGeneration, 0L, false, Optional.ofNullable(errorMessage)); } - static ServiceResponse createHostNotFoundInAppResponse(URI uri, String hostname, Long wantedGeneration) { - ServiceResponse serviceResponse = new ServiceResponse(410, uri, hostname, wantedGeneration); - serviceResponse.object.setString("problem", "Host:port (service) no longer part of application, refetch list of services."); - return serviceResponse; + private ServiceResponse(Status status, Long wantedGeneration, Long currentGeneration, boolean converged, Optional<String> errorMessage) { + this.status = status; + this.wantedGeneration = wantedGeneration; + this.currentGeneration = currentGeneration; + this.converged = converged; + this.errorMessage = errorMessage; } - static ServiceResponse createErrorResponse(URI uri, String hostname, Long wantedGeneration, String error) { - ServiceResponse serviceResponse = new ServiceResponse(500, uri, hostname, wantedGeneration); - serviceResponse.object.setString("error", error); - return serviceResponse; + } + + public static class ServiceListResponse { + + public final List<Service> services = new ArrayList<>(); + public final URI uri; + public final long wantedGeneration; + public final long currentGeneration; + + public ServiceListResponse(Map<ServiceInfo, Long> services, URI uri, long wantedGeneration, long currentGeneration) { + services.forEach((key, value) -> this.services.add(new Service(key, value))); + this.uri = uri; + this.wantedGeneration = wantedGeneration; + this.currentGeneration = currentGeneration; } - static ServiceResponse createNotFoundResponse(URI uri, String hostname, Long wantedGeneration, String error) { - ServiceResponse serviceResponse = new ServiceResponse(404, uri, hostname, wantedGeneration); - serviceResponse.object.setString("error", error); - return serviceResponse; + public List<Service> services() { return services; } + + public static class Service { + + public final ServiceInfo serviceInfo; + public final Long currentGeneration; + + public Service(ServiceInfo serviceInfo, Long currentGeneration) { + this.serviceInfo = serviceInfo; + this.currentGeneration = currentGeneration; + } + } + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index a77a8d1b5b8..5d944df2f30 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -184,7 +184,6 @@ public class ModelContextImpl implements ModelContext { private final int maxConcurrentMergesPerContentNode; private final int maxMergeQueueSize; private final boolean ignoreMergeQueueLimit; - private final int largeRankExpressionLimit; private final double resourceLimitDisk; private final double resourceLimitMemory; private final double minNodeRatioPerGroup; @@ -192,8 +191,6 @@ public class ModelContextImpl implements ModelContext { private final boolean containerDumpHeapOnShutdownTimeout; private final double containerShutdownTimeout; private final int distributorMergeBusyWait; - private final int docstoreCompressionLevel; - private final double diskBloatFactor; private final boolean distributorEnhancedMaintenanceScheduling; private final int maxUnCommittedMemory; private final boolean forwardIssuesAsErrors; @@ -204,9 +201,9 @@ public class ModelContextImpl implements ModelContext { private final boolean useV8DocManagerCfg; private final int maxCompactBuffers; private final boolean failDeploymentWithInvalidJvmOptions; - private final double tlsSizeFraction; private final List<String> ignoredHttpUserAgents; private final boolean enableServerOcspStapling; + private final String persistenceAsyncThrottling; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -226,7 +223,6 @@ public class ModelContextImpl implements ModelContext { this.allowedAthenzProxyIdentities = flagValue(source, appId, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS); this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); - this.largeRankExpressionLimit = flagValue(source, appId, Flags.LARGE_RANK_EXPRESSION_LIMIT); this.maxConcurrentMergesPerContentNode = flagValue(source, appId, Flags.MAX_CONCURRENT_MERGES_PER_NODE); this.maxMergeQueueSize = flagValue(source, appId, Flags.MAX_MERGE_QUEUE_SIZE); this.ignoreMergeQueueLimit = flagValue(source, appId, Flags.IGNORE_MERGE_QUEUE_LIMIT); @@ -237,8 +233,6 @@ public class ModelContextImpl implements ModelContext { this.containerDumpHeapOnShutdownTimeout = flagValue(source, appId, Flags.CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT); this.containerShutdownTimeout = flagValue(source, appId,Flags.CONTAINER_SHUTDOWN_TIMEOUT); this.distributorMergeBusyWait = flagValue(source, appId, Flags.DISTRIBUTOR_MERGE_BUSY_WAIT); - this.docstoreCompressionLevel = flagValue(source, appId, Flags.DOCSTORE_COMPRESSION_LEVEL); - this.diskBloatFactor = flagValue(source, appId, Flags.DISK_BLOAT_FACTOR); this.distributorEnhancedMaintenanceScheduling = flagValue(source, appId, Flags.DISTRIBUTOR_ENHANCED_MAINTENANCE_SCHEDULING); this.maxUnCommittedMemory = flagValue(source, appId, Flags.MAX_UNCOMMITTED_MEMORY);; this.forwardIssuesAsErrors = flagValue(source, appId, PermanentFlags.FORWARD_ISSUES_AS_ERRORS); @@ -249,9 +243,9 @@ public class ModelContextImpl implements ModelContext { this.useV8DocManagerCfg = flagValue(source, appId, Flags.USE_V8_DOC_MANAGER_CFG); this.maxCompactBuffers = flagValue(source, appId, Flags.MAX_COMPACT_BUFFERS); this.failDeploymentWithInvalidJvmOptions = flagValue(source, appId, Flags.FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS); - this.tlsSizeFraction = flagValue(source, appId, Flags.TLS_SIZE_FRACTION); this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS); this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING); + this.persistenceAsyncThrottling = flagValue(source, appId, Flags.PERSISTENCE_ASYNC_THROTTLING); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -273,7 +267,6 @@ public class ModelContextImpl implements ModelContext { @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return translateJvmOmitStackTraceInFastThrowIntToString(jvmOmitStackTraceInFastThrow, type); } - @Override public int largeRankExpressionLimit() { return largeRankExpressionLimit; } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerContentNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; } @@ -284,8 +277,6 @@ public class ModelContextImpl implements ModelContext { @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } - @Override public double diskBloatFactor() { return diskBloatFactor; } - @Override public int docstoreCompressionLevel() { return docstoreCompressionLevel; } @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } @Override public boolean forwardIssuesAsErrors() { return forwardIssuesAsErrors; } @@ -296,9 +287,9 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useV8DocManagerCfg() { return useV8DocManagerCfg; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } - @Override public double tlsSizeFraction() { return tlsSizeFraction; } @Override public List<String> ignoredHttpUserAgents() { return ignoredHttpUserAgents; } @Override public boolean enableServerOcspStapling() { return enableServerOcspStapling; } + @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index 81cd1dd9738..8e5eee2104c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -217,7 +218,8 @@ public class FileServer { return new FileDownloader(configServers.isEmpty() ? FileDownloader.emptyConnectionPool() : createConnectionPool(configServers, supervisor), - supervisor); + supervisor, + Duration.ofSeconds(10)); // set this low, to make sure we don't wait a for a long time in this thread } private static ConnectionPool createConnectionPool(List<String> configServers, Supervisor supervisor) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 4dda141491c..0131517818d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -5,6 +5,7 @@ import com.google.inject.Inject; import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.InstanceName; @@ -16,13 +17,15 @@ import com.yahoo.jdisc.Response; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; +import com.yahoo.slime.Cursor; import com.yahoo.slime.SlimeUtils; import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.application.ApplicationReindexing; +import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.http.ContentHandler; import com.yahoo.vespa.config.server.http.ContentRequest; import com.yahoo.vespa.config.server.http.HttpHandler; +import com.yahoo.vespa.config.server.http.JSONResponse; import com.yahoo.vespa.config.server.http.NotFoundException; import com.yahoo.vespa.config.server.http.v2.request.ApplicationContentRequest; import com.yahoo.vespa.config.server.http.v2.response.ApplicationSuspendedResponse; @@ -33,6 +36,7 @@ import com.yahoo.vespa.config.server.http.v2.response.ReindexingResponse; import com.yahoo.vespa.config.server.tenant.Tenant; import java.io.IOException; +import java.net.URI; import java.time.Duration; import java.time.Instant; import java.util.Map; @@ -43,8 +47,11 @@ import java.util.StringJoiner; import java.util.TreeMap; import java.util.TreeSet; +import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; +import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse; import static com.yahoo.yolean.Exceptions.uncheck; + /** * Operations on applications (delete, wait for config convergence, restart, application content etc.) * @@ -108,13 +115,21 @@ public class ApplicationHandler extends HttpHandler { } private HttpResponse listServiceConverge(ApplicationId applicationId, HttpRequest request) { - return applicationRepository.servicesToCheckForConfigConvergence(applicationId, request.getUri(), - getTimeoutFromRequest(request), getVespaVersionFromRequest(request)); + ServiceListResponse response = + applicationRepository.servicesToCheckForConfigConvergence(applicationId, + request.getUri(), + getTimeoutFromRequest(request), + getVespaVersionFromRequest(request)); + return new HttpServiceListResponse(response); } private HttpResponse checkServiceConverge(ApplicationId applicationId, String hostAndPort, HttpRequest request) { - return applicationRepository.checkServiceForConfigConvergence(applicationId, hostAndPort, request.getUri(), - getTimeoutFromRequest(request), getVespaVersionFromRequest(request)); + ServiceResponse response = + applicationRepository.checkServiceForConfigConvergence(applicationId, + hostAndPort, + getTimeoutFromRequest(request), + getVespaVersionFromRequest(request)); + return HttpServiceResponse.createResponse(response, hostAndPort, request.getUri()); } private HttpResponse serviceStatusPage(ApplicationId applicationId, String service, String hostname, String pathSuffix) { @@ -301,4 +316,79 @@ public class ApplicationHandler extends HttpHandler { .map(Version::fromString); } + static class HttpServiceResponse extends JSONResponse { + + public static HttpServiceResponse createResponse(ConfigConvergenceChecker.ServiceResponse serviceResponse, String hostAndPort, URI uri) { + switch (serviceResponse.status) { + case ok: + return createOkResponse(uri, hostAndPort, serviceResponse.wantedGeneration, serviceResponse.currentGeneration, serviceResponse.converged); + case hostNotFound: + return createHostNotFoundInAppResponse(uri, hostAndPort, serviceResponse.wantedGeneration); + case notFound: + return createNotFoundResponse(uri, hostAndPort, serviceResponse.wantedGeneration, serviceResponse.errorMessage.orElse("")); + case error: + return createErrorResponse(uri, hostAndPort, serviceResponse.wantedGeneration, serviceResponse.errorMessage.orElse("")); + default: + throw new IllegalArgumentException("Unknown status " + serviceResponse.status); + } + } + + private HttpServiceResponse(int status, URI uri, String hostname, Long wantedGeneration) { + super(status); + object.setString("url", uri.toString()); + object.setString("host", hostname); + object.setLong("wantedGeneration", wantedGeneration); + } + + private static HttpServiceResponse createOkResponse(URI uri, String hostname, Long wantedGeneration, Long currentGeneration, boolean converged) { + HttpServiceResponse serviceResponse = new HttpServiceResponse(200, uri, hostname, wantedGeneration); + serviceResponse.object.setBool("converged", converged); + serviceResponse.object.setLong("currentGeneration", currentGeneration); + return serviceResponse; + } + + private static HttpServiceResponse createHostNotFoundInAppResponse(URI uri, String hostname, Long wantedGeneration) { + HttpServiceResponse serviceResponse = new HttpServiceResponse(410, uri, hostname, wantedGeneration); + serviceResponse.object.setString("problem", "Host:port (service) no longer part of application, refetch list of services."); + return serviceResponse; + } + + private static HttpServiceResponse createErrorResponse(URI uri, String hostname, Long wantedGeneration, String error) { + HttpServiceResponse serviceResponse = new HttpServiceResponse(500, uri, hostname, wantedGeneration); + serviceResponse.object.setString("error", error); + return serviceResponse; + } + + private static HttpServiceResponse createNotFoundResponse(URI uri, String hostname, Long wantedGeneration, String error) { + HttpServiceResponse serviceResponse = new HttpServiceResponse(404, uri, hostname, wantedGeneration); + serviceResponse.object.setString("error", error); + return serviceResponse; + } + + } + + static class HttpServiceListResponse extends JSONResponse { + + // Pre-condition: servicesToCheck has a state port + public HttpServiceListResponse(ConfigConvergenceChecker.ServiceListResponse response) { + super(200); + Cursor serviceArray = object.setArray("services"); + response.services().forEach((service) -> { + ServiceInfo serviceInfo = service.serviceInfo; + Cursor serviceObject = serviceArray.addObject(); + String hostName = serviceInfo.getHostName(); + int statePort = ConfigConvergenceChecker.getStatePort(serviceInfo).get(); + serviceObject.setString("host", hostName); + serviceObject.setLong("port", statePort); + serviceObject.setString("type", serviceInfo.getServiceType()); + serviceObject.setString("url", response.uri.toString() + "/" + hostName + ":" + statePort); + serviceObject.setLong("currentGeneration", service.currentGeneration); + }); + object.setString("url", response.uri.toString()); + object.setLong("currentGeneration", response.currentGeneration); + object.setLong("wantedGeneration", response.wantedGeneration); + object.setBool("converged", response.currentGeneration >= response.wantedGeneration); + } + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index 47eabb0347e..a0e8d83fba1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -101,7 +101,7 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { ConnectionPool connectionPool = (otherConfigServersInCluster.isEmpty()) ? FileDownloader.emptyConnectionPool() : new FileDistributionConnectionPool(configSourceSet, supervisor); - return new FileDownloader(connectionPool, supervisor, downloadDirectory, Duration.ofSeconds(30)); + return new FileDownloader(connectionPool, supervisor, downloadDirectory, Duration.ofSeconds(300)); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index 687c897c88b..54ceb394ee6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -102,7 +102,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { ) { log.log(Level.FINE, () -> String.format("Loading model version %s for session %s application %s", modelFactory.version(), applicationGeneration, applicationId)); - ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId); + ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId, applicationPackage); Provisioned provisioned = new Provisioned(); ModelContext modelContext = new ModelContextImpl( applicationPackage, @@ -146,14 +146,14 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { return Optional.of(value); } - private ModelContext.Properties createModelContextProperties(ApplicationId applicationId) { + private ModelContext.Properties createModelContextProperties(ApplicationId applicationId, ApplicationPackage applicationPackage) { return new ModelContextImpl.Properties(applicationId, configserverConfig, zone(), ImmutableSet.copyOf(new ContainerEndpointsCache(TenantRepository.getTenantPath(tenant), curator).read(applicationId)), false, // We may be bootstrapping, but we only know and care during prepare false, // Always false, assume no one uses it when activating - flagSource, + LegacyFlags.from(applicationPackage, flagSource), new EndpointCertificateMetadataStore(curator, TenantRepository.getTenantPath(tenant)) .readEndpointCertificateMetadata(applicationId) .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/LegacyFlags.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/LegacyFlags.java new file mode 100644 index 00000000000..80467c80196 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/LegacyFlags.java @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.modelfactory; + +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.OrderedFlagSource; + +import java.util.Map; + + +/** + * @author arnej + */ +public class LegacyFlags { + + public static final String GEO_POSITIONS = "v7-geo-positions"; + public static final String FOO_BAR = "foo-bar"; // for testing + + private static FlagSource buildFrom(Map<String, String> legacyOverrides) { + var flags = new InMemoryFlagSource(); + for (var entry : legacyOverrides.entrySet()) { + String key = entry.getKey(); + String value = entry.getValue(); + boolean legacyWanted = Boolean.valueOf(value); + switch (key) { + case GEO_POSITIONS: + flags = flags.withBooleanFlag(Flags.USE_V8_GEO_POSITIONS.id(), ! legacyWanted); + break; + case FOO_BAR: + // ignored + break; + default: + throw new IllegalArgumentException("Unknown legacy override: "+key); + } + } + return flags; + } + + public static FlagSource from(ApplicationPackage pkg, FlagSource input) { + var overrides = buildFrom(pkg.legacyOverrides()); + FlagSource result = new OrderedFlagSource(overrides, input); + return result; + } +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index aaacc9f69e0..e4a0fa81f94 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -35,6 +35,7 @@ import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; import com.yahoo.vespa.config.server.filedistribution.FileDistributionFactory; import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.config.server.http.InvalidApplicationException; +import com.yahoo.vespa.config.server.modelfactory.LegacyFlags; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.modelfactory.PreparedModelsBuilder; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; @@ -195,7 +196,7 @@ public class SessionPreparer { Set.copyOf(containerEndpoints), params.isBootstrap(), currentActiveApplicationSet.isEmpty(), - flagSource, + LegacyFlags.from(applicationPackage, flagSource), endpointCertificateSecrets, athenzDomain, params.quota(), diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index c6a37e57e8a..42ccdffb2af 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -636,7 +636,7 @@ public class SessionRepository { .isAfter(clock.instant().minus(Duration.ofSeconds(30)))) newSessions.add(Long.parseLong(session.getName())); } catch (IOException e) { - log.log(Level.INFO, "Unable to find last modified time for " + session.toPath()); + log.log(Level.FINE, "Unable to find last modified time for " + session.toPath()); }; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 70e7dbe19ec..b3c2fa2e300 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -6,7 +6,6 @@ import com.google.inject.Inject; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.concurrent.DaemonThreadFactory; -import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.concurrent.Lock; import com.yahoo.concurrent.Locks; import com.yahoo.concurrent.StripedExecutor; @@ -36,7 +35,6 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; import org.apache.curator.framework.state.ConnectionState; @@ -202,7 +200,7 @@ public class TenantRepository { this.tenantListener = tenantListener; this.zookeeperServerConfig = zookeeperServerConfig; // This we should control with a feature flag. - this.deployHelperExecutor = createModelBuilderExecutor(Flags.NUM_DEPLOY_HELPER_THREADS.bindTo(flagSource).value()); + this.deployHelperExecutor = createModelBuilderExecutor(); curator.framework().getConnectionStateListenable().addListener(this::stateChanged); @@ -220,14 +218,11 @@ public class TenantRepository { TimeUnit.SECONDS); } - private ExecutorService createModelBuilderExecutor(int numThreads) { + private ExecutorService createModelBuilderExecutor() { final long GB = 1024*1024*1024; - if (numThreads == 0) return new InThreadExecutorService(); - if (numThreads < 0) { - long maxHeap = Runtime.getRuntime().maxMemory(); - int maxThreadsToFitInMemory = (int)((maxHeap + (GB - 1))/(1*GB)); - numThreads = Math.min(Runtime.getRuntime().availableProcessors(), maxThreadsToFitInMemory); - } + long maxHeap = Runtime.getRuntime().maxMemory(); + int maxThreadsToFitInMemory = (int)((maxHeap + (GB - 1))/(1*GB)); + int numThreads = Math.min(Runtime.getRuntime().availableProcessors(), maxThreadsToFitInMemory); return Executors.newFixedThreadPool(numThreads, ThreadFactoryFactory.getDaemonThreadFactory("deploy-helper")); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index 806b67758c2..6a483c38aee 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -10,6 +10,7 @@ import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.codegen.DefParser; +import com.yahoo.config.model.application.AbstractApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.serialization.AllocatedHostsSerializer; @@ -44,7 +45,7 @@ import static com.yahoo.vespa.config.server.zookeeper.ZKApplication.USERAPP_ZK_S * * @author Tony Vaagenes */ -public class ZKApplicationPackage implements ApplicationPackage { +public class ZKApplicationPackage extends AbstractApplicationPackage { private final ZKApplication zkApplication; diff --git a/configserver/src/test/apps/legacy-flag/schemas/music.sd b/configserver/src/test/apps/legacy-flag/schemas/music.sd new file mode 100644 index 00000000000..f4b11d1e8e4 --- /dev/null +++ b/configserver/src/test/apps/legacy-flag/schemas/music.sd @@ -0,0 +1,50 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# A basic search definition - called music, should be saved to music.sd +search music { + + # It contains one document type only - called music as well + document music { + + field title type string { + indexing: summary | index # How this field should be indexed + # index-to: title, default # Create two indexes + weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature + } + + field artist type string { + indexing: summary | attribute | index + # index-to: artist, default + + weight: 25 + } + + field year type int { + indexing: summary | attribute + } + + # Increase query + field popularity type int { + indexing: summary | attribute + } + + field url type uri { + indexing: summary | index + } + + } + + rank-profile default inherits default { + first-phase { + expression: nativeRank(title,artist) + attribute(popularity) + } + + } + + rank-profile textmatch inherits default { + first-phase { + expression: nativeRank(title,artist) + } + + } + +} diff --git a/configserver/src/test/apps/legacy-flag/services.xml b/configserver/src/test/apps/legacy-flag/services.xml new file mode 100644 index 00000000000..4c9ac84b4f2 --- /dev/null +++ b/configserver/src/test/apps/legacy-flag/services.xml @@ -0,0 +1,31 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <legacy> + <v7-geo-positions>false</v7-geo-positions> + <foo-bar>true</foo-bar> + </legacy> + + <admin version="2.0"> + <adminserver hostalias="node1"/> + <logserver hostalias="node1" /> + </admin> + + <content version="1.0"> + <redundancy>1</redundancy> + <documents> + <document type="music" mode="index"/> + </documents> + <nodes> + <node hostalias="node1" distribution-key="0"/> + </nodes> + </content> + + <container version="1.0"> + <nodes> + <node hostalias="node1" /> + </nodes> + </container> + +</services> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java index 8b21e9c3916..6afb9ef086d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ConfigConvergenceCheckerTest.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; -import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import org.junit.Before; @@ -16,22 +15,20 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UncheckedIOException; import java.net.URI; import java.time.Duration; import java.util.Arrays; -import java.util.function.Consumer; +import java.util.List; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.okJson; import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; -import static com.yahoo.test.json.JsonTestHelper.assertJsonEquals; -import static org.assertj.core.api.Assertions.assertThat; +import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; +import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -72,63 +69,35 @@ public class ConfigConvergenceCheckerTest { @Test public void service_convergence() { { // Known service - String serviceName = hostAndPort(this.service); - URI requestUrl = testServer().resolve("/serviceconverge/" + serviceName); wireMock.stubFor(get(urlEqualTo("/state/v1/config")).willReturn(okJson("{\"config\":{\"generation\":3}}"))); - HttpResponse serviceResponse = checker.getServiceConfigGenerationResponse(application, hostAndPort(this.service), requestUrl, clientTimeout); - assertResponse("{\n" + - " \"url\": \"" + requestUrl.toString() + "\",\n" + - " \"host\": \"" + hostAndPort(this.service) + "\",\n" + - " \"wantedGeneration\": 3,\n" + - " \"converged\": true,\n" + - " \"currentGeneration\": 3\n" + - "}", - 200, - serviceResponse); + + ServiceResponse response = checker.getServiceConfigGeneration(application, hostAndPort(this.service), clientTimeout); + assertEquals(3, response.wantedGeneration.longValue()); + assertEquals(3, response.currentGeneration.longValue()); + assertTrue(response.converged); + assertEquals(ServiceResponse.Status.ok, response.status); } { // Missing service - String serviceName = "notPresent:1337"; - URI requestUrl = testServer().resolve("/serviceconverge/" + serviceName); - HttpResponse response = checker.getServiceConfigGenerationResponse(application, "notPresent:1337", requestUrl, clientTimeout); - assertResponse("{\n" + - " \"url\": \"" + requestUrl.toString() + "\",\n" + - " \"host\": \"" + serviceName + "\",\n" + - " \"wantedGeneration\": 3,\n" + - " \"problem\": \"Host:port (service) no longer part of application, refetch list of services.\"\n" + - "}", - 410, - response); + ServiceResponse response = checker.getServiceConfigGeneration(application, "notPresent:1337", clientTimeout); + assertEquals(3, response.wantedGeneration.longValue()); + assertEquals(ServiceResponse.Status.hostNotFound, response.status); } } @Test public void service_list_convergence() { { - String serviceName = hostAndPort(this.service); URI requestUrl = testServer().resolve("/serviceconverge"); - URI serviceUrl = testServer().resolve("/serviceconverge/" + serviceName); wireMock.stubFor(get(urlEqualTo("/state/v1/config")).willReturn(okJson("{\"config\":{\"generation\":3}}"))); - HttpResponse response = checker.getServiceConfigGenerationsResponse(application, requestUrl, clientTimeout); - assertResponse("{\n" + - " \"services\": [\n" + - " {\n" + - " \"host\": \"" + serviceUrl.getHost() + "\",\n" + - " \"port\": " + serviceUrl.getPort() + ",\n" + - " \"type\": \"container\",\n" + - " \"url\": \"" + serviceUrl.toString() + "\",\n" + - " \"currentGeneration\":" + 3 + "\n" + - " }\n" + - " ],\n" + - " \"url\": \"" + requestUrl.toString() + "\",\n" + - " \"currentGeneration\": 3,\n" + - " \"wantedGeneration\": 3,\n" + - " \"converged\": true\n" + - "}", - 200, - response); - } + ServiceListResponse response = checker.getServiceConfigGenerations(application, requestUrl, clientTimeout); + assertEquals(3, response.wantedGeneration); + assertEquals(3, response.currentGeneration); + List<ServiceListResponse.Service> services = response.services; + assertEquals(1, services.size()); + assertService(this.service, services.get(0), 3); + } { // Model with two hosts on different generations MockModel model = new MockModel(Arrays.asList( @@ -143,53 +112,29 @@ public class ConfigConvergenceCheckerTest { wireMock2.stubFor(get(urlEqualTo("/state/v1/config")).willReturn(okJson("{\"config\":{\"generation\":3}}"))); URI requestUrl = testServer().resolve("/serviceconverge"); - URI serviceUrl = testServer().resolve("/serviceconverge/" + hostAndPort(service)); - URI serviceUrl2 = testServer().resolve("/serviceconverge/" + hostAndPort(service2)); - HttpResponse response = checker.getServiceConfigGenerationsResponse(application, requestUrl, clientTimeout); - assertResponse("{\n" + - " \"services\": [\n" + - " {\n" + - " \"host\": \"" + service.getHost() + "\",\n" + - " \"port\": " + service.getPort() + ",\n" + - " \"type\": \"container\",\n" + - " \"url\": \"" + serviceUrl.toString() + "\",\n" + - " \"currentGeneration\":" + 4 + "\n" + - " },\n" + - " {\n" + - " \"host\": \"" + service2.getHost() + "\",\n" + - " \"port\": " + service2.getPort() + ",\n" + - " \"type\": \"container\",\n" + - " \"url\": \"" + serviceUrl2.toString() + "\",\n" + - " \"currentGeneration\":" + 3 + "\n" + - " }\n" + - " ],\n" + - " \"url\": \"" + requestUrl.toString() + "\",\n" + - " \"currentGeneration\": 3,\n" + - " \"wantedGeneration\": 4,\n" + - " \"converged\": false\n" + - "}", - 200, - response); + + ServiceListResponse response = checker.getServiceConfigGenerations(application, requestUrl, clientTimeout); + assertEquals(4, response.wantedGeneration); + assertEquals(3, response.currentGeneration); + + List<ServiceListResponse.Service> services = response.services; + assertEquals(2, services.size()); + assertService(this.service, services.get(0), 4); + assertService(this.service2, services.get(1), 3); } } + @Test public void service_convergence_timeout() { - URI requestUrl = testServer().resolve("/serviceconverge"); wireMock.stubFor(get(urlEqualTo("/state/v1/config")).willReturn(aResponse() .withFixedDelay((int) clientTimeout.plus(Duration.ofSeconds(1)).toMillis()) .withBody("response too slow"))); - HttpResponse response = checker.getServiceConfigGenerationResponse(application, hostAndPort(service), requestUrl, Duration.ofMillis(1)); - // Message contained in a SocketTimeoutException may differ across platforms, so we do a partial match of the response here - assertResponse( - responseBody -> - assertThat(responseBody) - .startsWith("{\"url\":\"" + requestUrl.toString() + "\",\"host\":\"" + hostAndPort(requestUrl) + - "\",\"wantedGeneration\":3,\"error\":\"") - .contains("java.net.SocketTimeoutException: 1 MILLISECONDS") - .endsWith("\"}"), - 404, - response); + ServiceResponse response = checker.getServiceConfigGeneration(application, hostAndPort(service), Duration.ofMillis(1)); + + assertEquals(3, response.wantedGeneration.longValue()); + assertEquals(ServiceResponse.Status.notFound, response.status); + assertTrue(response.errorMessage.get().contains("java.net.SocketTimeoutException: 1 MILLISECONDS")); } private URI testServer() { @@ -204,19 +149,11 @@ public class ConfigConvergenceCheckerTest { return uri.getHost() + ":" + uri.getPort(); } - private static void assertResponse(String expectedJson, int status, HttpResponse response) { - assertResponse((responseBody) -> assertJsonEquals(new String(responseBody.getBytes()), expectedJson), status, response); - } - - private static void assertResponse(Consumer<String> assertFunc, int status, HttpResponse response) { - ByteArrayOutputStream responseBody = new ByteArrayOutputStream(); - try { - response.render(responseBody); - assertFunc.accept(responseBody.toString()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - assertEquals(status, response.getStatus()); + private void assertService(URI uri, ServiceListResponse.Service service1, long expectedGeneration) { + assertEquals(expectedGeneration, service1.currentGeneration.longValue()); + assertEquals(uri.getHost(), service1.serviceInfo.getHostName()); + assertEquals(uri.getPort(), ConfigConvergenceChecker.getStatePort(service1.serviceInfo).get().intValue()); + assertEquals("container", service1.serviceInfo.getServiceType()); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/LegacyFlagsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/LegacyFlagsTest.java new file mode 100644 index 00000000000..1bebe9089f1 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/LegacyFlagsTest.java @@ -0,0 +1,68 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.application; + +import com.yahoo.vespa.config.server.modelfactory.LegacyFlags; +import com.yahoo.cloud.config.ModelConfig; +import com.yahoo.component.Version; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.ModelStub; +import com.yahoo.vespa.config.server.ServerCache; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; +import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.model.VespaModel; +import org.junit.Before; +import org.junit.Test; + +import com.yahoo.config.ConfigInstance; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.buildergen.ConfigDefinition; +import com.yahoo.document.config.DocumentmanagerConfig; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author arnej + */ +public class LegacyFlagsTest { + + @Test + public void testThatLegacyOverridesWork() throws Exception { + File testApp = new File("src/test/apps/legacy-flag"); + var appPkg = FilesApplicationPackage.fromFile(testApp); + var flag = Flags.USE_V8_GEO_POSITIONS.bindTo(LegacyFlags.from(appPkg, new InMemoryFlagSource())); + assertTrue(flag.value()); + /* rest here tests that having a "legacy" XML tag doesn't break other things, but without actually using it: */ + VespaModel model = new VespaModel(appPkg); + ApplicationId applicationId = new ApplicationId.Builder().tenant("foo").applicationName("foo").build(); + ServerCache cache = new ServerCache(); + Application app = new Application(model, cache, 1L, new Version(1, 2, 3), + new MetricUpdater(Metrics.createTestMetrics(), Metrics.createDimensions(applicationId)), applicationId); + assertNotNull(app.getModel()); + /* + // Note: no feature flags active with this code path + ConfigDefinitionKey cdk = new ConfigDefinitionKey(DocumentmanagerConfig.CONFIG_DEF_NAME, DocumentmanagerConfig.CONFIG_DEF_NAMESPACE); + ConfigDefinition cdd = new ConfigDefinition(cdk.getName(), DocumentmanagerConfig.CONFIG_DEF_SCHEMA); + var cfgBuilder = app.getModel().getConfigInstance(new ConfigKey<>(cdk.getName(), "client", cdk.getNamespace()), cdd); + assertTrue(cfgBuilder instanceof DocumentmanagerConfig.Builder); + var cfg = ((DocumentmanagerConfig.Builder)cfgBuilder).build(); + // no effect from legacy override seen here: + System.out.println("CFG: "+cfg); + */ + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 0b8bf8e84bd..04483e0191d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.config.server.http.v2; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.PortInfo; +import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -50,14 +52,18 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.function.Consumer; import java.util.stream.Stream; import static com.yahoo.container.jdisc.HttpRequest.createTestRequest; @@ -65,8 +71,12 @@ import static com.yahoo.jdisc.http.HttpRequest.Method.DELETE; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; import static com.yahoo.jdisc.http.HttpRequest.Method.POST; import static com.yahoo.test.json.JsonTestHelper.assertJsonEquals; +import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceListResponse; +import static com.yahoo.vespa.config.server.application.ConfigConvergenceChecker.ServiceResponse; import static com.yahoo.vespa.config.server.http.HandlerTest.assertHttpStatusCodeAndMessage; import static com.yahoo.vespa.config.server.http.SessionHandlerTest.getRenderedString; +import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServiceListResponse; +import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServiceResponse.createResponse; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -146,7 +156,7 @@ public class ApplicationHandlerTest { Tenant mytenant = applicationRepository.getTenant(applicationId); deleteAndAssertOKResponse(mytenant, applicationId); } - + { applicationRepository.deploy(testApp, prepareParams(applicationId)); deleteAndAssertOKResponseMocked(applicationId, true); @@ -539,6 +549,143 @@ public class ApplicationHandlerTest { "}\n"); } + @Test + public void service_convergence() { + String hostAndPort = "localhost:1234"; + URI uri = URI.create("https://" + hostAndPort + "/serviceconvergence/container"); + + { // Known service + HttpResponse response = createResponse(new ServiceResponse(ServiceResponse.Status.ok, + 3L, + 3L, + true), + hostAndPort, + uri); + assertResponse("{\n" + + " \"url\": \"" + uri.toString() + "\",\n" + + " \"host\": \"" + hostAndPort + "\",\n" + + " \"wantedGeneration\": 3,\n" + + " \"converged\": true,\n" + + " \"currentGeneration\": 3\n" + + "}", + 200, + response); + } + + { // Missing service + HttpResponse response = createResponse(new ServiceResponse(ServiceResponse.Status.hostNotFound, + 3L), + hostAndPort, + uri); + + assertResponse("{\n" + + " \"url\": \"" + uri.toString() + "\",\n" + + " \"host\": \"" + hostAndPort + "\",\n" + + " \"wantedGeneration\": 3,\n" + + " \"problem\": \"Host:port (service) no longer part of application, refetch list of services.\"\n" + + "}", + 410, + response); + } + } + + @Test + public void service_list_convergence() { + URI requestUrl = URI.create("https://configserver/serviceconvergence"); + + String hostname = "localhost"; + int port = 1234; + String hostAndPort = hostname + ":" + port; + URI serviceUrl = URI.create("https://configserver/serviceconvergence/" + hostAndPort); + + { + HttpServiceListResponse response = + new HttpServiceListResponse(new ServiceListResponse(Map.of(createServiceInfo(hostname, port), 3L), + requestUrl, + 3L, + 3L)); + assertResponse("{\n" + + " \"services\": [\n" + + " {\n" + + " \"host\": \"" + hostname + "\",\n" + + " \"port\": " + port + ",\n" + + " \"type\": \"container\",\n" + + " \"url\": \"" + serviceUrl.toString() + "\",\n" + + " \"currentGeneration\":" + 3 + "\n" + + " }\n" + + " ],\n" + + " \"url\": \"" + requestUrl.toString() + "\",\n" + + " \"currentGeneration\": 3,\n" + + " \"wantedGeneration\": 3,\n" + + " \"converged\": true\n" + + "}", + 200, + response); + } + + { // Two hosts on different generations + String hostname2 = "localhost2"; + int port2 = 5678; + String hostAndPort2 = hostname2 + ":" + port2; + URI serviceUrl2 = URI.create("https://configserver/serviceconvergence/" + hostAndPort2); + + Map<ServiceInfo, Long> serviceInfos = new HashMap<>(); + serviceInfos.put(createServiceInfo(hostname, port), 4L); + serviceInfos.put(createServiceInfo(hostname2, port2), 3L); + + HttpServiceListResponse response = + new HttpServiceListResponse(new ServiceListResponse(serviceInfos, + requestUrl, + 4L, + 3L)); + assertResponse("{\n" + + " \"services\": [\n" + + " {\n" + + " \"host\": \"" + hostname + "\",\n" + + " \"port\": " + port + ",\n" + + " \"type\": \"container\",\n" + + " \"url\": \"" + serviceUrl.toString() + "\",\n" + + " \"currentGeneration\":" + 4 + "\n" + + " },\n" + + " {\n" + + " \"host\": \"" + hostname2 + "\",\n" + + " \"port\": " + port2 + ",\n" + + " \"type\": \"container\",\n" + + " \"url\": \"" + serviceUrl2.toString() + "\",\n" + + " \"currentGeneration\":" + 3 + "\n" + + " }\n" + + " ],\n" + + " \"url\": \"" + requestUrl.toString() + "\",\n" + + " \"currentGeneration\": 3,\n" + + " \"wantedGeneration\": 4,\n" + + " \"converged\": false\n" + + "}", + 200, + response); + } + } + + @Test + public void service_convergence_timeout() { + String hostAndPort = "localhost:1234"; + URI uri = URI.create("https://" + hostAndPort + "/serviceconvergence/container"); + + HttpResponse response = createResponse(new ServiceResponse(ServiceResponse.Status.notFound, + 3L, + "some error message"), + hostAndPort, + uri); + + assertResponse("{\n" + + " \"url\": \"" + uri.toString() + "\",\n" + + " \"host\": \"" + hostAndPort + "\",\n" + + " \"wantedGeneration\": 3,\n" + + " \"error\": \"some error message\"" + + "}", + 404, + response); + } + private void assertNotAllowed(Method method) throws IOException { String url = "http://myhost:14000/application/v2/tenant/" + mytenantName + "/application/default"; deleteAndAssertResponse(url, Response.Status.METHOD_NOT_ALLOWED, HttpErrorResponse.ErrorCode.METHOD_NOT_ALLOWED, "{\"error-code\":\"METHOD_NOT_ALLOWED\",\"message\":\"Method '" + method + "' is not supported\"}", @@ -668,4 +815,29 @@ public class ApplicationHandlerTest { return new PrepareParams.Builder().applicationId(applicationId).build(); } + private static void assertResponse(String expectedJson, int status, HttpResponse response) { + assertResponse((responseBody) -> assertJsonEquals(new String(responseBody + .getBytes()), expectedJson), status, response); + } + + private static void assertResponse(Consumer<String> assertFunc, int status, HttpResponse response) { + ByteArrayOutputStream responseBody = new ByteArrayOutputStream(); + try { + response.render(responseBody); + assertFunc.accept(responseBody.toString()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + assertEquals(status, response.getStatus()); + } + + private ServiceInfo createServiceInfo(String hostname, int port) { + return new ServiceInfo("container", + "container", + List.of(new PortInfo(port, List.of("state"))), + Map.of(), + "configId", + hostname); + } + } diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index d5be3ab52f2..bb8317c298b 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -2667,6 +2667,18 @@ "public static final enum com.yahoo.metrics.simple.UntypedMetric$AssumedType COUNTER" ] }, + "com.yahoo.metrics.simple.UntypedMetric$Histogram": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public double getValueAtPercentile(double)", + "public void outputPercentileDistribution(java.io.PrintStream, int, java.lang.Double, boolean)" + ], + "fields": [] + }, "com.yahoo.metrics.simple.UntypedMetric": { "superClass": "java.lang.Object", "interfaces": [], @@ -2680,7 +2692,7 @@ "public double getMax()", "public double getMin()", "public double getSum()", - "public org.HdrHistogram.DoubleHistogram getHistogram()", + "public com.yahoo.metrics.simple.UntypedMetric$Histogram getHistogram()", "public java.lang.String toString()" ], "fields": [] diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java b/container-core/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java index ad549fb4d91..3d82e7853d9 100644 --- a/container-core/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java +++ b/container-core/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.metrics.simple; -import java.util.logging.Logger; - +import com.yahoo.api.annotations.Beta; import org.HdrHistogram.DoubleHistogram; +import java.io.PrintStream; import java.util.logging.Level; +import java.util.logging.Logger; /** * A gauge or a counter or... who knows? The class for storing a metric when the @@ -114,8 +115,9 @@ public class UntypedMetric { return metricSettings; } - public DoubleHistogram getHistogram() { - return histogram; + @Beta + public Histogram getHistogram() { + return histogram != null ? new Histogram(histogram) : null; } @Override @@ -139,4 +141,19 @@ public class UntypedMetric { return buf.toString(); } + @Beta + public static class Histogram { + private final DoubleHistogram hdrHistogram; + + private Histogram(DoubleHistogram hdrHistogram) { this.hdrHistogram = hdrHistogram; } + + public double getValueAtPercentile(double percentile) { return hdrHistogram.getValueAtPercentile(percentile); } + + public void outputPercentileDistribution(PrintStream printStream, int percentileTicksPerHalfDistance, + Double outputValueUnitScalingRatio, boolean useCsvFormat) { + hdrHistogram.outputPercentileDistribution( + printStream, percentileTicksPerHalfDistance, outputValueUnitScalingRatio, useCsvFormat); + } + } + } diff --git a/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java index 5b5fb67f1b4..4cd0d820433 100644 --- a/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java +++ b/container-core/src/main/java/com/yahoo/metrics/simple/jdisc/SnapshotConverter.java @@ -1,21 +1,30 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.metrics.simple.jdisc; -import java.io.PrintStream; -import java.util.*; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -import org.HdrHistogram.DoubleHistogram; - import com.yahoo.collections.Tuple2; -import com.yahoo.container.jdisc.state.*; +import com.yahoo.container.jdisc.state.CountMetric; +import com.yahoo.container.jdisc.state.GaugeMetric; +import com.yahoo.container.jdisc.state.MetricDimensions; +import com.yahoo.container.jdisc.state.MetricSet; +import com.yahoo.container.jdisc.state.MetricSnapshot; +import com.yahoo.container.jdisc.state.MetricValue; +import com.yahoo.container.jdisc.state.StateMetricContext; import com.yahoo.metrics.simple.Bucket; import com.yahoo.metrics.simple.Identifier; import com.yahoo.metrics.simple.Point; import com.yahoo.metrics.simple.UntypedMetric; +import com.yahoo.metrics.simple.UntypedMetric.Histogram; import com.yahoo.metrics.simple.Value; -import com.yahoo.text.JSON; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; /** * Convert simple metrics snapshots into jdisc state snapshots. @@ -74,7 +83,7 @@ class SnapshotConverter { } } - private static List<Tuple2<String, Double>> buildPercentileList(DoubleHistogram histogram) { + private static List<Tuple2<String, Double>> buildPercentileList(Histogram histogram) { List<Tuple2<String, Double>> prefixAndValues = new ArrayList<>(2); prefixAndValues.add(new Tuple2<>("95", histogram.getValueAtPercentile(95.0d))); prefixAndValues.add(new Tuple2<>("99", histogram.getValueAtPercentile(99.0d))); @@ -122,7 +131,7 @@ class SnapshotConverter { continue; } gotHistogram = true; - DoubleHistogram histogram = entry.getValue().getHistogram(); + Histogram histogram = entry.getValue().getHistogram(); Identifier id = entry.getKey(); String metricIdentifier = getIdentifierString(id); output.println("# start of metric " + metricIdentifier); diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java index 7fa294422d6..7571cfb0969 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java @@ -43,6 +43,7 @@ public final class MbusClient extends AbstractResource implements ClientProvider this.session = session; this.sessionReference = session.refer(this); thread = new Thread(new SenderTask(), "mbus-client-" + threadId.getAndIncrement()); + thread.setDaemon(true); } @Override @@ -79,6 +80,11 @@ public final class MbusClient extends AbstractResource implements ClientProvider log.log(Level.FINE, "Destroying message bus client."); sessionReference.close(); done = true; + try { + thread.join(); + } catch (InterruptedException e) { + log.log(Level.WARNING, "Interrupted while joining thread on destroy.", e); + } } @Override @@ -121,7 +127,7 @@ public final class MbusClient extends AbstractResource implements ClientProvider if (error == null) { return true; } - if (error.isFatal()) { + if (error.isFatal() || done) { final Reply reply = new EmptyReply(); reply.swapState(request.getMessage()); reply.addError(error); diff --git a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java index e7b1fc3e71d..9a8dda246a0 100644 --- a/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java +++ b/container-messagebus/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java @@ -75,9 +75,7 @@ public final class MbusServer extends AbstractResource implements ServerProvider return; } if (state == State.STOPPED) { - // We might need to detect requests originating from the same JVM, as they nede to fail fast - // As they are holding references to the container preventing proper shutdown. - dispatchErrorReply(msg, ErrorCode.SESSION_BUSY, "MBusServer has been closed."); + dispatchErrorReply(msg, ErrorCode.NETWORK_SHUTDOWN, "MBusServer has been closed."); return; } if (msg.getTrace().shouldTrace(6)) { diff --git a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java index 397a3eb5242..4ef26534a92 100644 --- a/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java +++ b/container-messagebus/src/test/java/com/yahoo/messagebus/jdisc/MbusServerConformanceTest.java @@ -30,7 +30,7 @@ import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import static com.yahoo.messagebus.ErrorCode.APP_FATAL_ERROR; -import static com.yahoo.messagebus.ErrorCode.SESSION_BUSY; +import static com.yahoo.messagebus.ErrorCode.NETWORK_SHUTDOWN; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -50,7 +50,7 @@ public class MbusServerConformanceTest extends ServerProviderConformanceTest { @Test public void testContainerNotReadyException() throws Throwable { new TestRunner().setRequestTimeout(100, TimeUnit.MILLISECONDS) - .expectError(SESSION_BUSY) + .expectError(NETWORK_SHUTDOWN) .executeAndClose(); } diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 6ed01c2a998..b320a1090ae 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -1740,7 +1740,7 @@ "public java.lang.Integer addToken(long, int)", "public java.lang.Integer addToken(java.lang.String, int)", "public java.lang.Integer addToken(java.lang.String)", - "public java.lang.Integer getTokenWeight(java.lang.String)", + "public java.lang.Integer getTokenWeight(java.lang.Object)", "public java.lang.Integer removeToken(java.lang.String)", "public int getNumTokens()", "public java.util.Iterator getTokens()", @@ -1896,7 +1896,8 @@ "public static final enum com.yahoo.search.Query$Type WEB", "public static final enum com.yahoo.search.Query$Type PROGRAMMATIC", "public static final enum com.yahoo.search.Query$Type YQL", - "public static final enum com.yahoo.search.Query$Type SELECT" + "public static final enum com.yahoo.search.Query$Type SELECT", + "public static final enum com.yahoo.search.Query$Type WEAKAND" ] }, "com.yahoo.search.Query": { @@ -5537,7 +5538,6 @@ "public void setLocale(java.lang.String, com.yahoo.search.query.Sorting$UcaSorter$Strength)", "public java.lang.String getLocale()", "public com.yahoo.search.query.Sorting$UcaSorter$Strength getStrength()", - "public com.ibm.icu.text.Collator getCollator()", "public java.lang.String getDecomposition()", "public java.lang.String toSerialForm()", "public int hashCode()", diff --git a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java index aaa4d33c6dc..d3fbeb020f8 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/CompositeItem.java @@ -80,11 +80,6 @@ public abstract class CompositeItem extends Item { subitems.add(index, item); } - /** For NOT items, which may wish to insert nulls */ - void insertNullFirstItem() { - subitems.add(0, null); - } - /** * Returns a subitem * @@ -109,7 +104,7 @@ public abstract class CompositeItem extends Item { adding(item); Item old = subitems.set(index, item); - if (old!=item) + if (old != item) removing(old); return old; } @@ -188,9 +183,7 @@ public abstract class CompositeItem extends Item { return itemCount; } - /** - * Encodes just this item, not it's usual subitems, to the given buffer. - */ + /** Encodes just this item, not its regular subitems, to the given buffer. */ protected void encodeThis(ByteBuffer buffer) { super.encodeThis(buffer); IntegerCompressor.putCompressedPositiveNumber(encodingArity(), buffer); @@ -279,10 +272,7 @@ public abstract class CompositeItem extends Item { return code; } - /** - * Returns whether this item is of the same class and - * contains the same state as the given item - */ + /** Returns whether this item is of the same class and contains the same state as the given item. */ @Override public boolean equals(Object object) { if (!super.equals(object)) return false; @@ -303,17 +293,12 @@ public abstract class CompositeItem extends Item { @Override public int getTermCount() { int terms = 0; - for (Item item : subitems) { + for (Item item : subitems) terms += item.getTermCount(); - } return terms; } - /** - * Will return its single child if itself can safely be omitted. - * - * @return a valid Item or empty Optional if it can not be done - */ + /** Returns the single child of this, if this can be omitted without changes to recall semantics. */ public Optional<Item> extractSingleChild() { return getItemCount() == 1 ? Optional.of(getItem(0)) : Optional.empty(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NotItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NotItem.java index 833b8635f61..6985ed0913c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/NotItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/NotItem.java @@ -7,11 +7,12 @@ import java.util.Objects; /** * A composite item where the first item is positive and the following - * items are negative items which should be excluded from the result. + * items are negative items where matches should exclude the document should from the result. + * The default positive item, if only negatives are added, is TrueItem: Meaning that all documents are matched + * except those matching the negative terms added. * * @author bratseth */ -// TODO: Handle nulls by creating nullItem or checking in encode/toString public class NotItem extends CompositeItem { @Override @@ -25,6 +26,7 @@ public class NotItem extends CompositeItem { } /** Adds an item. The first item is the positive, the rest are negative */ + @Override public void addItem(Item item) { super.addItem(item); } @@ -34,27 +36,25 @@ public class NotItem extends CompositeItem { * (position 0) if it is not already set. */ public void addNegativeItem(Item negative) { - if (getItemCount() == 0) { - insertNullFirstItem(); - } + if (getItemCount() == 0) + insertTrueFirstItem(); addItem(negative); } /** Returns the negative items of this: All child items except the first */ public List<Item> negativeItems() { return items().subList(1, getItemCount()); } - /** Returns the positive item (the first subitem), or null if no positive items has been added. */ + /** Returns the positive item (the first subitem), or TrueItem if no positive items has been added. */ public Item getPositiveItem() { - if (getItemCount() == 0) { - return null; - } + if (getItemCount() == 0) + return new TrueItem(); return getItem(0); } /** * Sets the positive item (the first item) * - * @return the old positive item, or null if there was none + * @return the old positive item, or TrueItem if there was none */ public Item setPositiveItem(Item item) { Objects.requireNonNull(item, () -> "Positive item of " + this); @@ -72,7 +72,7 @@ public class NotItem extends CompositeItem { * the positive item becomes an AndItem with the items added */ public void addPositiveItem(Item item) { - if (getPositiveItem() == null) { + if (getPositiveItem() instanceof TrueItem) { setPositiveItem(item); } else if (getPositiveItem() instanceof AndItem) { ((AndItem) getPositiveItem()).addItem(item); @@ -90,7 +90,7 @@ public class NotItem extends CompositeItem { boolean removed = super.removeItem(item); if (removed && removedIndex == 0) { - insertNullFirstItem(); + insertTrueFirstItem(); } return removed; } @@ -99,35 +99,35 @@ public class NotItem extends CompositeItem { Item removed = super.removeItem(index); if (index == 0) { // Don't make the first negative the positive - insertNullFirstItem(); + insertTrueFirstItem(); } return removed; } + private void insertTrueFirstItem() { + addItem(0, new TrueItem()); + } + /** Not items uses a empty heading instead of "NOT " */ protected void appendHeadingString(StringBuilder buffer) {} /** - * Overridden to tolerate nulls and to append "+" + * Overridden to skip the positive TrueItem and (otherwise) append "+" * to the first item and "-" to the rest */ + @Override protected void appendBodyString(StringBuilder buffer) { - boolean isFirstItem = true; - - for (Iterator<Item> i = getItemIterator(); i.hasNext();) { - Item item = i.next(); - - if (isFirstItem) { - buffer.append("+"); - } else { - buffer.append(" -"); - } - if (item == null) { - buffer.append("(null)"); - } else { - buffer.append(item.toString()); - } - isFirstItem = false; + if (items().isEmpty()) return; + if (items().size() == 1) { + buffer.append(items().get(0)); + return; + } + for (int i = 0; i < items().size(); i++) { + if (i == 0 && items().get(i) instanceof TrueItem) continue; // skip positive true + + buffer.append(i == 0 ? "+" : "-").append(items().get(i)); + if ( i < items().size() - 1) + buffer.append(" "); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java index 1f30833b3db..8c4c5c84a28 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/QueryCanonicalizer.java @@ -77,11 +77,6 @@ public class QueryCanonicalizer { else if (composite instanceof RankItem) { makeDuplicatesCheap((RankItem)composite); } - else if (composite instanceof NotItem) { - if (((NotItem) composite).getPositiveItem() == null) - return CanonicalizationResult.error("Can not search for only negative items"); - } - if (composite.getItemCount() == 0) parentIterator.remove(); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java b/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java index 4770a02e51a..f71f25821ad 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/SimpleTaggableItem.java @@ -72,4 +72,5 @@ public abstract class SimpleTaggableItem extends Item implements TaggableItem { public boolean hasUniqueID() { return super.hasUniqueID(); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java b/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java index e75a8417328..a988753d699 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/WeightedSetItem.java @@ -22,7 +22,6 @@ import java.util.Objects; * contain the weights of all the matched tokens in descending * order. Each matched weight will be represented as a standard * occurrence on position 0 in element 0. - * */ public class WeightedSetItem extends SimpleTaggableItem { @@ -79,7 +78,7 @@ public class WeightedSetItem extends SimpleTaggableItem { return addToken(token, 1); } - public Integer getTokenWeight(String token) { + public Integer getTokenWeight(Object token) { return set.get(token); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java index 3759dbbcbee..80a2320b039 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.query.parser; import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; import com.yahoo.prelude.query.IntItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.NotItem; @@ -10,6 +11,7 @@ import com.yahoo.prelude.query.OrItem; import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.QueryCanonicalizer; import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.WeakAndItem; import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.parser.ParserEnvironment; @@ -26,8 +28,16 @@ import static com.yahoo.prelude.query.parser.Token.Kind.SPACE; */ public class AllParser extends SimpleParser { - public AllParser(ParserEnvironment environment) { + private final boolean weakAnd; + + /** + * Creates an And parser + * + * @param weakAnd false to parse into AndItem (by default), true to parse to WeakAnd + */ + public AllParser(ParserEnvironment environment, boolean weakAnd) { super(environment); + this.weakAnd = weakAnd; } @Override @@ -42,7 +52,7 @@ public class AllParser extends SimpleParser { protected Item parseItemsBody() { // Algorithm: Collect positive, negative, and and'ed items, then combine. - AndItem and = null; + CompositeItem and = null; NotItem not = null; // Store negatives here as we go Item current; @@ -88,13 +98,17 @@ public class AllParser extends SimpleParser { return root.getRoot() instanceof NullItem ? null : root.getRoot(); } - protected AndItem addAnd(Item item, AndItem and) { + protected CompositeItem addAnd(Item item, CompositeItem and) { if (and == null) - and = new AndItem(); + and = createAnd(); and.addItem(item); return and; } + private CompositeItem createAnd() { + return weakAnd ? new WeakAndItem() : new AndItem(); + } + protected OrItem addOr(Item item, OrItem or) { if (or == null) or = new OrItem(); @@ -124,7 +138,7 @@ public class AllParser extends SimpleParser { if (item != null) { isComposited = true; if (item instanceof OrItem) { // Turn into And - AndItem and = new AndItem(); + CompositeItem and = createAnd(); for (Iterator<Item> i = ((OrItem) item).getItemIterator(); i.hasNext();) { and.addItem(i.next()); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index 087da13a937..020d93d951c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -121,7 +121,7 @@ abstract class SimpleParser extends StructuredParser { return combineItems(topLevelItem, not.getPositiveItem()); } } - if (not != null && not.getPositiveItem() == null) { + if (not != null && not.getPositiveItem() instanceof TrueItem) { // Incomplete not, only negatives - if (topLevelItem != null && topLevelItem != not) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java index a7dbaa94fc0..d7c7dec4798 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java @@ -1,11 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; -import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.WordItem; import com.yahoo.search.query.parser.ParserEnvironment; -import java.util.Set; - /** * Parser for web search queries. Language: * @@ -21,37 +24,38 @@ import java.util.Set; public class WebParser extends AllParser { public WebParser(ParserEnvironment environment) { - super(environment); + super(environment, false); } - protected @Override Item parseItemsBody() { + @Override + protected Item parseItemsBody() { // Algorithm: Collect positive, negative, and'ed and or'ed elements, then combine. - AndItem and=null; - OrItem or=null; - NotItem not=null; // Store negatives here as we go + CompositeItem and = null; + OrItem or = null; + NotItem not = null; // Store negatives here as we go Item current; // Find all items do { - current=negativeItem(); - if (current!=null) { - not=addNot(current,not); + current = negativeItem(); + if (current != null) { + not = addNot(current, not); continue; } - current=positiveItem(); - if (current==null) + current = positiveItem(); + if (current == null) current = indexableItem(); - if (current!=null) { - if (and!=null && (current instanceof WordItem) && "OR".equals(((WordItem)current).getRawWord())) { - if (or==null) - or=addOr(and,or); - and=new AndItem(); + if (current != null) { + if (and != null && (current instanceof WordItem) && "OR".equals(((WordItem)current).getRawWord())) { + if (or == null) + or = addOr(and, or); + and = new AndItem(); or.addItem(and); } else { - and=addAnd(current,and); + and = addAnd(current, and); } } @@ -60,21 +64,17 @@ public class WebParser extends AllParser { } while (tokens.hasNext()); // Combine the items - Item topLevel=and; + Item topLevel = and; - if (or!=null) - topLevel=or; + if (or != null) + topLevel = or; - if (not!=null && topLevel!=null) { + if (not != null && topLevel != null) { not.setPositiveItem(topLevel); - topLevel=not; + topLevel = not; } return simplifyUnnecessaryComposites(topLevel); } - protected void setSubmodeFromIndex(String indexName, Set<String> searchDefinitions) { - // No submodes in this language - } - } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/LiteralBoostSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/LiteralBoostSearcher.java index e9e5818cefe..47a5213c041 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/LiteralBoostSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/LiteralBoostSearcher.java @@ -71,8 +71,6 @@ public class LiteralBoostSearcher extends Searcher { } private void addLiterals(RankItem rankTerms, Item item, IndexFacts.Session indexFacts) { - if (item == null) return; - if (item instanceof NotItem) { addLiterals(rankTerms, ((NotItem) item).getPositiveItem(), indexFacts); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java index 2b8515b6db8..8e137d99951 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleBase.java @@ -1,19 +1,34 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics; +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.StemMode; +import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; +import com.yahoo.prelude.semantics.rule.CompositeCondition; +import com.yahoo.prelude.semantics.rule.Condition; +import com.yahoo.prelude.semantics.rule.NamedCondition; +import com.yahoo.prelude.semantics.rule.ProductionRule; +import com.yahoo.prelude.semantics.rule.SuperCondition; import com.yahoo.search.Query; import com.yahoo.prelude.querytransform.PhraseMatcher; import com.yahoo.prelude.semantics.engine.RuleEngine; import com.yahoo.prelude.semantics.parser.ParseException; -import com.yahoo.prelude.semantics.rule.*; import com.yahoo.protect.Validator; import java.io.File; -import java.util.*; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; +import java.util.StringTokenizer; /** - * A set of semantic production rules and named conditions used to analyze - * and rewrite queries + * A set of semantic production rules and named conditions used to analyze and rewrite queries * * @author bratseth */ @@ -26,7 +41,7 @@ public class RuleBase { private String source; /** The name of the automata file used, or null if none */ - protected String automataFileName = null; + private String automataFileName = null; /** * True if this rule base is default. @@ -61,29 +76,26 @@ public class RuleBase { */ private boolean usesAutomata = false; - /** Should we allow stemmed matches? */ - private boolean stemming = true; - - /** Creates an empty rule base. TODO: Disallow */ - public RuleBase() { - } + private RuleBaseLinguistics linguistics; /** Creates an empty rule base */ - public RuleBase(String name) { - setName(name); + public RuleBase(String name, Linguistics linguistics) { + this.name = name; + this.linguistics = new RuleBaseLinguistics(StemMode.BEST, Language.ENGLISH, linguistics); } /** - * Creates a rule base from a file + * Creates a rule base from file * - * @param ruleFile the rule file to read. The name of the file (minus path) becomes the rule base name + * @param ruleFile the rule file to read. The name of the file (minus path) becomes the rule base name. * @param automataFile the automata file, or null to not use an automata * @throws java.io.IOException if there is a problem reading one of the files * @throws ParseException if the rule file can not be parsed correctly * @throws RuleBaseException if the rule file contains inconsistencies */ - public static RuleBase createFromFile(String ruleFile, String automataFile) throws java.io.IOException, ParseException { - return new RuleImporter().importFile(ruleFile, automataFile); + public static RuleBase createFromFile(String ruleFile, String automataFile, Linguistics linguistics) + throws java.io.IOException, ParseException { + return new RuleImporter(linguistics).importFile(ruleFile, automataFile); } /** @@ -96,18 +108,13 @@ public class RuleBase { * @throws com.yahoo.prelude.semantics.parser.ParseException if the rule file can not be parsed correctly * @throws com.yahoo.prelude.semantics.RuleBaseException if the rule file contains inconsistencies */ - public static RuleBase createFromString(String name, String ruleString, String automataFile) throws java.io.IOException, ParseException { - RuleBase base = new RuleImporter().importString(ruleString, automataFile, new RuleBase()); + public static RuleBase createFromString(String name, String ruleString, String automataFile, Linguistics linguistics) + throws java.io.IOException, ParseException { + RuleBase base = new RuleImporter(linguistics).importString(ruleString, automataFile); base.setName(name); return base; } - /** Set to true to enable stemmed matches. True by default */ - public void setStemming(boolean stemming) { this.stemming = stemming; } - - /** Returns whether stemmed matches are allowed. True by default */ - public boolean getStemming() { return stemming; } - /** * <p>Include another rule base into this. This <b>transfers ownership</b> * of the given rule base - it can not be subsequently used for any purpose @@ -171,7 +178,7 @@ public class RuleBase { resolveSuper(condition, superCondition); } - private void resolveSuper(Condition condition,Condition superCondition) { + private void resolveSuper(Condition condition, Condition superCondition) { if (condition instanceof SuperCondition) { ((SuperCondition)condition).setCondition(superCondition); } @@ -336,7 +343,7 @@ public class RuleBase { // TODO: Values are not added right now protected void annotatePhrase(PhraseMatcher.Phrase phrase,Query query,int traceLevel) { - for (StringTokenizer tokens = new StringTokenizer(phrase.getData(),"|",false) ; tokens.hasMoreTokens(); ) { + for (StringTokenizer tokens = new StringTokenizer(phrase.getData(), "|", false); tokens.hasMoreTokens(); ) { String token = tokens.nextToken(); int semicolonIndex = token.indexOf(";"); String annotation = token; diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java index 45569050882..acbf9a7ffb6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/RuleImporter.java @@ -10,8 +10,9 @@ import java.util.Arrays; import java.util.List; import com.yahoo.io.IOUtils; -import com.yahoo.io.reader.NamedReader; -import com.yahoo.prelude.semantics.parser.*; +import com.yahoo.language.Linguistics; +import com.yahoo.prelude.semantics.parser.ParseException; +import com.yahoo.prelude.semantics.parser.SemanticsParser; /** * Imports rule bases from various sources. @@ -24,51 +25,47 @@ import com.yahoo.prelude.semantics.parser.*; // rule bases included into others, while neither the rule base or the parser knows. public class RuleImporter { - /** - * If this is set, imported rule bases are looked up in this config - * otherwise, they are looked up as files - */ - private SemanticRulesConfig config; + /** If this is set, imported rule bases are looked up in this config otherwise, they are looked up as files. */ + private final SemanticRulesConfig config; - /** - * Ignore requests to read automata files. - * Useful to validate rule bases without having automatas present - */ - private boolean ignoreAutomatas; + /** Ignore requests to read automata files. Useful to validate rule bases without having automatas present. */ + private final boolean ignoreAutomatas; - /** - * Ignore requests to include files. - * Useful to validate rule bases one by one in config - */ - private boolean ignoreIncludes = false; + /** Ignore requests to include files. Useful to validate rule bases one by one in config. */ + private final boolean ignoreIncludes; + + private Linguistics linguistics; /** Create a rule importer which will read from file */ - public RuleImporter() { - this(null, false); + public RuleImporter(Linguistics linguistics) { + this(null, false, linguistics); } /** Create a rule importer which will read from a config object */ - public RuleImporter(SemanticRulesConfig config) { - this(config, false); + public RuleImporter(SemanticRulesConfig config, Linguistics linguistics) { + this(config, false, linguistics); } - public RuleImporter(boolean ignoreAutomatas) { - this(null, ignoreAutomatas); + public RuleImporter(boolean ignoreAutomatas, Linguistics linguistics) { + this(null, ignoreAutomatas, linguistics); } - public RuleImporter(boolean ignoreAutomatas, boolean ignoreIncludes) { - this(null, ignoreAutomatas, ignoreIncludes); + public RuleImporter(boolean ignoreAutomatas, boolean ignoreIncludes, Linguistics linguistics) { + this(null, ignoreAutomatas, ignoreIncludes, linguistics); } - public RuleImporter(SemanticRulesConfig config, boolean ignoreAutomatas) { - this.config = config; - this.ignoreAutomatas = ignoreAutomatas; + public RuleImporter(SemanticRulesConfig config, boolean ignoreAutomatas, Linguistics linguistics) { + this(config, ignoreAutomatas, false, linguistics); } - public RuleImporter(SemanticRulesConfig config, boolean ignoreAutomatas, boolean ignoreIncludes) { + public RuleImporter(SemanticRulesConfig config, + boolean ignoreAutomatas, + boolean ignoreIncludes, + Linguistics linguistics) { this.config = config; this.ignoreAutomatas = ignoreAutomatas; this.ignoreIncludes = ignoreIncludes; + this.linguistics = linguistics; } /** @@ -91,33 +88,18 @@ public class RuleImporter { * @throws ParseException if the file does not contain a valid semantic rule set */ public RuleBase importFile(String fileName, String automataFile) throws IOException, ParseException { - return importFile(fileName, automataFile, null); - } - - /** - * Imports semantic rules from a file - * - * @param fileName the rule file to use - * @param automataFile the automata file to use, or null to not use any - * @param ruleBase an existing rule base to import these rules into, or null to create a new - * @throws java.io.IOException if the file can not be read for some reason - * @throws ParseException if the file does not contain a valid semantic rule set - */ - public RuleBase importFile(String fileName, String automataFile, RuleBase ruleBase) throws IOException, ParseException { - ruleBase = privateImportFile(fileName, automataFile, ruleBase); + var ruleBase = privateImportFile(fileName, automataFile); ruleBase.initialize(); return ruleBase; } - public RuleBase privateImportFile(String fileName, String automataFile, RuleBase ruleBase) throws IOException, ParseException { + public RuleBase privateImportFile(String fileName, String automataFile) throws IOException, ParseException { BufferedReader reader = null; try { reader = IOUtils.createReader(fileName, "utf-8"); File file = new File(fileName); String absoluteFileName = file.getAbsolutePath(); - if (ruleBase == null) - ruleBase = new RuleBase(); - ruleBase.setName(stripLastName(file.getName())); + var ruleBase = new RuleBase(stripLastName(file.getName()), linguistics); privateImportFromReader(reader, absoluteFileName, automataFile, ruleBase); return ruleBase; } @@ -157,18 +139,17 @@ public class RuleImporter { /** Returns an unitialized rule base */ private RuleBase privateImportFromDirectory(String ruleBaseName, RuleBase ruleBase) throws IOException, ParseException { - RuleBase include = new RuleBase(); String includeDir = new File(ruleBase.getSource()).getParentFile().getAbsolutePath(); if (!ruleBaseName.endsWith(".sr")) ruleBaseName = ruleBaseName + ".sr"; File importFile = new File(includeDir, ruleBaseName); if ( ! importFile.exists()) throw new IOException("No file named '" + shortenPath(importFile.getPath()) + "'"); - return privateImportFile(importFile.getPath(), null, include); + return privateImportFile(importFile.getPath(), null); } /** Returns an unitialized rule base */ - private RuleBase privateImportFromConfig(String ruleBaseName) throws IOException, ParseException { + private RuleBase privateImportFromConfig(String ruleBaseName) throws ParseException { SemanticRulesConfig.Rulebase ruleBaseConfig = findRuleBaseConfig(config,ruleBaseName); if (ruleBaseConfig == null) ruleBaseConfig = findRuleBaseConfig(config, stripLastName(ruleBaseName)); @@ -224,8 +205,7 @@ public class RuleImporter { /** Imports an unitialized rule base */ public RuleBase privateImportConfig(SemanticRulesConfig.Rulebase ruleBaseConfig) throws ParseException { if (config == null) throw new IllegalStateException("Must initialize with config if importing from config"); - RuleBase ruleBase = new RuleBase(); - ruleBase.setName(ruleBaseConfig.name()); + RuleBase ruleBase = new RuleBase(ruleBaseConfig.name(), linguistics); return privateImportFromReader(new StringReader(ruleBaseConfig.rules()), "semantic-rules.cfg", ruleBaseConfig.automata(),ruleBase); @@ -253,14 +233,10 @@ public class RuleImporter { /** Returns an unitialized rule base */ public RuleBase privateImportFromReader(Reader reader, String sourceName, String automataFile, RuleBase ruleBase) throws ParseException { try { - if (ruleBase == null) { - ruleBase = new RuleBase(); - if (sourceName == null) - sourceName = "anonymous"; - ruleBase.setName(sourceName); - } + if (ruleBase == null) + ruleBase = new RuleBase(sourceName == null ? "anonymous" : sourceName, linguistics); ruleBase.setSource(sourceName.replace('\\', '/')); - new SemanticsParser(reader).semanticRules(ruleBase, this); + new SemanticsParser(reader, linguistics).semanticRules(ruleBase, this); if (automataFile != null && !automataFile.isEmpty()) ruleBase.setAutomataFile(automataFile.replace('\\', '/')); return ruleBase; diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java index f9d968a3a4d..a8167fd2001 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java @@ -4,6 +4,7 @@ package com.yahoo.prelude.semantics; import com.google.inject.Inject; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; +import com.yahoo.language.Linguistics; import com.yahoo.prelude.ConfigurationException; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -13,7 +14,9 @@ import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; -import java.util.*; +import java.util.Arrays; +import java.util.List; +import java.util.Map; import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; @@ -38,7 +41,7 @@ public class SemanticSearcher extends Searcher { /** Creates a semantic searcher using the given default rule base */ public SemanticSearcher(RuleBase ruleBase) { - this(Collections.singletonList(ruleBase)); + this(List.of(ruleBase)); defaultRuleBase = ruleBase; } @@ -47,8 +50,8 @@ public class SemanticSearcher extends Searcher { } @Inject - public SemanticSearcher(SemanticRulesConfig config) { - this(toList(config)); + public SemanticSearcher(SemanticRulesConfig config, Linguistics linguistics) { + this(toList(config, linguistics)); } public SemanticSearcher(List<RuleBase> ruleBases) { @@ -59,9 +62,9 @@ public class SemanticSearcher extends Searcher { } } - private static List<RuleBase> toList(SemanticRulesConfig config) { + private static List<RuleBase> toList(SemanticRulesConfig config, Linguistics linguistics) { try { - RuleImporter ruleImporter = new RuleImporter(config); + RuleImporter ruleImporter = new RuleImporter(config, linguistics); List<RuleBase> ruleBaseList = new java.util.ArrayList<>(); for (SemanticRulesConfig.Rulebase ruleBaseConfig : config.rulebase()) { RuleBase ruleBase = ruleImporter.importConfig(ruleBaseConfig); diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/benchmark/RuleBaseBenchmark.java b/container-search/src/main/java/com/yahoo/prelude/semantics/benchmark/RuleBaseBenchmark.java index 938d12b271b..75b6e831983 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/benchmark/RuleBaseBenchmark.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/benchmark/RuleBaseBenchmark.java @@ -9,6 +9,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.Iterator; +import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.search.Query; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleImporter; @@ -27,7 +28,7 @@ public class RuleBaseBenchmark { fsaFile = null; } } - RuleBase ruleBase = new RuleImporter().importFile(ruleBaseFile,fsaFile); + RuleBase ruleBase = new RuleImporter(new SimpleLinguistics()).importFile(ruleBaseFile, fsaFile); ArrayList<String> queries = new ArrayList<>(); BufferedReader reader = new BufferedReader(new FileReader(queryFile)); String line; @@ -35,7 +36,7 @@ public class RuleBaseBenchmark { queries.add(line); } Date start = new Date(); - for (int i=0;i<iterations;i++){ + for (int i=0; i<iterations; i++){ for (Iterator<String> iter = queries.iterator(); iter.hasNext(); ){ String queryString = iter.next(); Query query = new Query("?query="+queryString); @@ -43,7 +44,7 @@ public class RuleBaseBenchmark { } } Date end = new Date(); - long elapsed = end.getTime()-start.getTime(); + long elapsed = end.getTime() - start.getTime(); System.out.print("BENCHMARK: rulebase=" + ruleBaseFile + "\n fsa=" + fsaFile + "\n queries=" + queryFile + diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleBaseLinguistics.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleBaseLinguistics.java new file mode 100644 index 00000000000..c5519632d6d --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleBaseLinguistics.java @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.engine; + +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.StemList; +import com.yahoo.language.process.StemMode; + +import java.util.List; +import java.util.Objects; + +/** + * Linguistics for a rule base + * + * @author bratseth + */ +public class RuleBaseLinguistics { + + private final StemMode stemMode; + private final Language language; + private final Linguistics linguistics; + + /** Creates a rule base with default settings */ + public RuleBaseLinguistics(Linguistics linguistics) { + this(StemMode.BEST, Language.ENGLISH, linguistics); + } + + + public RuleBaseLinguistics(StemMode stemMode, Language language, Linguistics linguistics) { + this.stemMode = Objects.requireNonNull(stemMode); + this.language = Objects.requireNonNull(language); + this.linguistics = Objects.requireNonNull(linguistics); + } + + public RuleBaseLinguistics withStemMode(StemMode stemMode) { + return new RuleBaseLinguistics(stemMode, language, linguistics); + } + + public RuleBaseLinguistics withLanguage(Language language) { + return new RuleBaseLinguistics(stemMode, language, linguistics); + } + + public Linguistics linguistics() { return linguistics; } + + /** Processes this term according to the linguistics of this rule base */ + public String process(String term) { + if (stemMode == StemMode.NONE) return term; + List<StemList> stems = linguistics.getStemmer().stem(term, StemMode.BEST, language); + if (stems.isEmpty()) return term; + if (stems.get(0).isEmpty()) return term; + return stems.get(0).get(0); + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java index e7ed05730cb..dd6610d1184 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/engine/RuleEngine.java @@ -17,7 +17,7 @@ import java.util.ListIterator; */ public class RuleEngine { - private RuleBase rules; + private final RuleBase rules; public RuleEngine(RuleBase rules) { this.rules=rules; @@ -38,7 +38,6 @@ public class RuleEngine { boolean matchedAnything = false; Evaluation evaluation = new Evaluation(query, traceLevel); - evaluation.setStemming(rules.getStemming()); if (traceLevel >= 2) evaluation.trace(2,"Evaluating query '" + evaluation.getQuery().getModel().getQueryTree().getRoot() + "':"); for (ListIterator<ProductionRule> i = rules.ruleIterator(); i.hasNext(); ) { diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralCondition.java index 42bf0560726..b85dd892047 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/LiteralCondition.java @@ -4,7 +4,7 @@ package com.yahoo.prelude.semantics.rule; import com.yahoo.prelude.semantics.engine.RuleEvaluation; /** - * A condition which is always true, and which has it's own value as return value + * A condition which is always true, and which has its own value as return value * * @author bratseth */ diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java index b2592a36353..a267d274d5a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamedCondition.java @@ -14,9 +14,9 @@ public class NamedCondition { private Condition condition; - public NamedCondition(String name,Condition condition) { - this.conditionName=name; - this.condition=condition; + public NamedCondition(String name, Condition condition) { + this.conditionName = name; + this.condition = condition; } public String getName() { return conditionName; } @@ -28,18 +28,18 @@ public class NamedCondition { public void setCondition(Condition condition) { this.condition = condition; } public boolean matches(RuleEvaluation e) { - if (e.getTraceLevel()>=3) { + if (e.getTraceLevel() >= 3) { e.trace(3,"Evaluating '" + this + "' at " + e.currentItem()); e.indentTrace(); } boolean matches=condition.matches(e); - if (e.getTraceLevel()>=3) { + if (e.getTraceLevel() >= 3) { e.unindentTrace(); if (matches) e.trace(3,"Matched '" + this + "' at " + e.previousItem()); - else if (e.getTraceLevel()>=4) + else if (e.getTraceLevel() >= 4) e.trace(4,"Did not match '" + this + "' at " + e.currentItem()); } return matches; diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamespaceProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamespaceProduction.java index 099a8562ece..e6f32a83dd9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamespaceProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/NamespaceProduction.java @@ -18,13 +18,13 @@ public class NamespaceProduction extends Production { private String key; /** The value to set in the namespace */ - private String value=null; + private String value; /** Creates a produced template term with no label and the default type */ - public NamespaceProduction(String namespace,String key,String value) { + public NamespaceProduction(String namespace, String key, String value) { setNamespace(namespace); - this.key=key; - this.value=value; + this.key = key; + this.value = value; } public String getNamespace() { return namespace; } @@ -44,7 +44,7 @@ public class NamespaceProduction extends Production { public void setValue(String value) { this.value = value; } - public void produce(RuleEvaluation e,int offset) { + public void produce(RuleEvaluation e, int offset) { e.getEvaluation().getQuery().properties().set(key, value); } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java index b36744dc397..af7abf325e7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/ReferenceTermProduction.java @@ -12,7 +12,7 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; import com.yahoo.protect.Validator; /** - * A term produced by a production rule which takes it's actual term value + * A term produced by a production rule which takes its actual term value * from one or more terms matched in the condition * * @author bratseth diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java index 38d1fc9b83b..a2bbf72a53b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermCondition.java @@ -3,6 +3,7 @@ package com.yahoo.prelude.semantics.rule; import com.yahoo.prelude.query.TermItem; import com.yahoo.prelude.semantics.engine.NameSpace; +import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.prelude.semantics.engine.RuleEvaluation; /** @@ -12,42 +13,35 @@ import com.yahoo.prelude.semantics.engine.RuleEvaluation; */ public class TermCondition extends Condition { - private String term, termPlusS; + private final RuleBaseLinguistics linguistics; + private final String originalTerm; + private final String term; - /** Creates an invalid term */ - public TermCondition() { } - - public TermCondition(String term) { - this(null,term); + public TermCondition(String term, RuleBaseLinguistics linguistics) { + this(null, term, linguistics); } - public TermCondition(String label, String term) { + public TermCondition(String label, String term, RuleBaseLinguistics linguistics) { super(label); - this.term = term; - termPlusS = term + "s"; - } - - public String getTerm() { return term; } - - public void setTerm(String term) { - this.term = term; - termPlusS = term + "s"; + this.linguistics = linguistics; + this.originalTerm = term; + this.term = linguistics.process(term); } protected boolean doesMatch(RuleEvaluation e) { // TODO: Move this into the respective namespaces when query becomes one */ if (getNameSpace() != null) { NameSpace nameSpace = e.getEvaluation().getNameSpace(getNameSpace()); - return nameSpace.matches(term, e); + return nameSpace.matches(originalTerm, e); // No processing of terms in namespaces } else { if (e.currentItem() == null) return false; if ( ! labelMatches(e)) return false; - String matchedValue = termMatches(e.currentItem().getItem(), e.getEvaluation().getStemming()); - boolean matches = matchedValue!=null && labelMatches(e.currentItem().getItem(), e); + boolean matches = labelMatches(e.currentItem().getItem(), e) && + linguistics.process(e.currentItem().getItem().stringValue()).equals(term); if ((matches && !e.isInNegation() || (!matches && e.isInNegation()))) { - e.addMatch(e.currentItem(), matchedValue); + e.addMatch(e.currentItem(), originalTerm); e.setValue(term); e.next(); } @@ -55,34 +49,6 @@ public class TermCondition extends Condition { } } - /** Returns a non-null replacement term if there is a match, null otherwise */ - private String termMatches(TermItem queryTerm, boolean stemming) { - String queryTermString = queryTerm.stringValue(); - - // The terms are the same - boolean matches = queryTermString.equals(term); - if (matches) return term; - - if (stemming) - if (termMatchesWithStemming(queryTermString)) return term; - - return null; - } - - private boolean termMatchesWithStemming(String queryTermString) { - if (queryTermString.length() < 3) return false; // Don't stem very short terms - - // The query term minus s is the same - boolean matches = queryTermString.equals(termPlusS); - if (matches) return true; - - // The query term plus s is the same - matches = term.equals(queryTermString + "s"); - if (matches) return true; - - return false; - } - public String toInnerString() { return getLabelString() + term; } diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java index db8d4b42521..29e4982ac17 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/rule/TermProduction.java @@ -15,7 +15,7 @@ import com.yahoo.protect.Validator; public abstract class TermProduction extends Production { /** The label of this term, or null if none */ - private String label = null; + private String label; /** The type of term to produce */ private TermType termType; @@ -62,7 +62,7 @@ public abstract class TermProduction extends Production { protected void insertMatch(RuleEvaluation e, Match matched, Item newItem, int offset) { if (getWeight() != 100) newItem.setWeight(getWeight()); - int insertPosition = matched.getPosition()+offset; + int insertPosition = matched.getPosition() + offset; // This check is necessary (?) because earlier items may have been removed // after we recorded the match position. It is sort of hackish. A cleaner diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 786a0d0e04f..623c38fa9f0 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -108,7 +108,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { WEB(4,"web"), PROGRAMMATIC(5, "prog"), YQL(6, "yql"), - SELECT(7, "select"); + SELECT(7, "select"), + WEAKAND(8, "weakAnd"); private final int intValue; private final String stringValue; @@ -123,7 +124,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { for (Type type : Type.values()) if (type.stringValue.equals(typeString)) return type; - return ALL; + throw new IllegalArgumentException("No query type '" + typeString + "'"); } public int asInt() { return intValue; } diff --git a/container-search/src/main/java/com/yahoo/search/query/Sorting.java b/container-search/src/main/java/com/yahoo/search/query/Sorting.java index a98ee44cb59..f1da48c1e08 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Sorting.java +++ b/container-search/src/main/java/com/yahoo/search/query/Sorting.java @@ -323,7 +323,7 @@ public class Sorting implements Cloneable { public String getLocale() { return locale; } public Strength getStrength() { return strength; } - public Collator getCollator() { return collator; } + Collator getCollator() { return collator; } public String getDecomposition() { return (collator.getDecomposition() == Collator.CANONICAL_DECOMPOSITION) ? "CANONICAL_DECOMPOSITION" : "NO_DECOMPOSITION"; } @Override diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java index a64af7658cc..f9b8f1785db 100644 --- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java +++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java @@ -29,7 +29,7 @@ public final class ParserFactory { public static Parser newInstance(Query.Type type, ParserEnvironment environment) { switch (type) { case ALL: - return new AllParser(environment); + return new AllParser(environment, false); case ANY: return new AnyParser(environment); case PHRASE: @@ -44,6 +44,8 @@ public final class ParserFactory { return new YqlParser(environment); case SELECT: return new SelectParser(environment); + case WEAKAND: + return new AllParser(environment, true); default: throw new UnsupportedOperationException(type.toString()); } diff --git a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/AndNotRestConverter.java b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/AndNotRestConverter.java index b64809e8071..795a78157c5 100644 --- a/container-search/src/main/java/com/yahoo/search/query/textserialize/item/AndNotRestConverter.java +++ b/container-search/src/main/java/com/yahoo/search/query/textserialize/item/AndNotRestConverter.java @@ -13,6 +13,7 @@ import static com.yahoo.search.query.textserialize.item.ListUtil.first; * @author Tony Vaagenes */ public class AndNotRestConverter extends CompositeConverter<NotItem> { + static final String andNotRest = "AND-NOT-REST"; public AndNotRestConverter() { @@ -51,4 +52,5 @@ public class AndNotRestConverter extends CompositeConverter<NotItem> { protected String getFormName(Item item) { return andNotRest; } + } diff --git a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java index 72a8b02a960..b1d64329927 100644 --- a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java +++ b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java @@ -23,6 +23,8 @@ import java.util.Set; /** * A wrapper for structured data representing feature values: A map of floats and tensors. * This class is immutable but not thread safe. + * + * @author bratseth */ public class FeatureData implements Inspectable, JsonProducer { diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java index f81083221a8..94b7c140b0f 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/ValidateNearestNeighborSearcher.java @@ -102,7 +102,7 @@ public class ValidateNearestNeighborSearcher extends Searcher { String queryFeatureName = "query(" + item.getQueryTensorName() + ")"; Optional<Tensor> queryTensor = query.getRanking().getFeatures().getTensor(queryFeatureName); if (queryTensor.isEmpty()) - return item + " requires a tensor rank feature " + queryFeatureName + " but this is not present"; + return item + " requires a tensor rank feature named '" + queryFeatureName + "' but this is not present"; if ( ! validAttributes.containsKey(item.getIndexName())) { return item + " field is not an attribute"; diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java index 649d678db55..93cac27059e 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java +++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java @@ -18,6 +18,7 @@ import com.yahoo.search.query.parser.ParserFactory; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; +import com.yahoo.yolean.Exceptions; import com.yahoo.yolean.chain.After; import com.yahoo.yolean.chain.Before; import com.yahoo.yolean.chain.Provides; @@ -93,7 +94,9 @@ public class MinimalQueryInserter extends Searcher { Parsable parsable = Parsable.fromQueryModel(query.getModel()).setQuery(query.properties().getString(YQL)); newTree = parser.parse(parsable); } catch (RuntimeException e) { - return new Result(query, ErrorMessage.createInvalidQueryParameter("Could not instantiate query from YQL", e)); + return new Result(query, ErrorMessage.createInvalidQueryParameter("Could not create query from YQL: " + + Exceptions.toMessageString(e), + e)); } if (parser.getOffset() != null) { int maxHits = query.properties().getInteger(MAX_HITS); diff --git a/container-search/src/main/java/com/yahoo/search/yql/ParameterListParser.java b/container-search/src/main/java/com/yahoo/search/yql/ParameterListParser.java new file mode 100644 index 00000000000..5a609f0025b --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/yql/ParameterListParser.java @@ -0,0 +1,204 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import com.yahoo.prelude.query.WeightedSetItem; + +import java.util.Arrays; + +/** + * Parser of parameter lists on the form {key:value, key:value} or [[key,value], [key,value], ...] + * + * @author bratseth + */ +class ParameterListParser { + + public static void addItemsFromString(String string, WeightedSetItem out) { + var s = new ParsableString(string); + switch (s.peek()) { + case '[' : addArrayItems(s, out); break; + case '{' : addMapItems(s, out); break; + default : throw new IllegalArgumentException("Expected a string starting by '[' or '{', " + + "but was '" + s.peek() + "'"); + } + } + + private static void addArrayItems(ParsableString s, WeightedSetItem out) { + s.pass('['); + while (s.peek() != ']') { + s.pass('['); + long key = s.longTo(s.position(',')); + s.pass(','); + int value = s.intTo(s.position(']')); + s.pass(']'); + out.addToken(key, value); + s.passOptional(','); + if (s.atEnd()) throw new IllegalArgumentException("Expected an array ending by ']'"); + } + s.pass(']'); + } + + private static void addMapItems(ParsableString s, WeightedSetItem out) { + s.pass('{'); + while (s.peek() != '}') { + String key; + if (s.passOptional('\'')) { + key = s.stringTo(s.position('\'')); + s.pass('\''); + } + else if (s.passOptional('"')) { + key = s.stringTo(s.position('"')); + s.pass('"'); + } + else { + key = s.stringTo(s.position(':')).trim(); + } + s.pass(':'); + int value = s.intTo(s.position(',','}')); + out.addToken(key, value); + s.passOptional(','); + if (s.atEnd()) throw new IllegalArgumentException("Expected a map ending by '}'"); + } + s.pass('}'); + } + + private static class ParsableString { + + int position = 0; + String s; + + ParsableString(String s) { + this.s = s; + } + + /** + * Returns the next non-space character or UNASSIGNED if we have reached the end of the string. + * The current position is not changed. + */ + char peek() { + int localPosition = position; + while (localPosition < s.length()) { + char nextChar = s.charAt(localPosition++); + if (!Character.isSpaceChar(nextChar)) + return nextChar; + } + return Character.UNASSIGNED; + } + + /** + * Verifies that the next non-space character is the given and moves the position past it. + * + * @throws IllegalArgumentException if the next non-space character is not the given character + */ + void pass(char character) { + while (position < s.length()) { + char nextChar = s.charAt(position++); + if (!Character.isSpaceChar(nextChar)) { + if (nextChar == character) + return; + else + throw new IllegalArgumentException("Expected '" + character + "' at position " + (position-1) + + " but got '" + nextChar + "'"); + } + } + throw new IllegalArgumentException("Expected '" + character + "' at position " + (position-1) + + " but reached the end"); + } + + /** + * Checks if the next non-space character is the given and moves the position past it if so. + * Does not change the position otherwise. + * + * @return true if the next non-space character was the given character + */ + boolean passOptional(char character) { + int localPosition = position; + while (localPosition < s.length()) { + char nextChar = s.charAt(localPosition++); + if (!Character.isSpaceChar(nextChar)) { + if (nextChar == character) { + position = localPosition; + return true; + } else { + return false; + } + } + } + return false; + } + + /** + * Returns the position of the next occurrence of any of the given characters. + * + * @throws IllegalArgumentException if there are no further occurrences of any of the given characters + */ + int position(char ... characters) { + int localPosition = position; + while (localPosition < s.length()) { + char nextChar = s.charAt(localPosition); + for (char character : characters) + if (nextChar == character) return localPosition; + localPosition++; + } + throw new IllegalArgumentException("Expected one of " + Arrays.toString(characters) + " after " + position); + } + + boolean atEnd() { + return position >= s.length(); + } + + /** + * Returns the string value from the current to the given position, and moves the current + * position to the next character. + * + * @throws IllegalArgumentException if end is beyond the last position of the string + */ + String stringTo(int end) { + try { + String value = s.substring(position, end); + position = end; + return value; + } + catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException(end + " is larger than the size of the string, " + s.length()); + } + } + + /** + * Returns the int value from the current to the given position, and moves the current + * position to the next character. + * + * @throws IllegalArgumentException if the string cannot be parsed to an int or end is larger than the string + */ + int intTo(int end) { + int start = position; + String value = stringTo(end); + try { + return Integer.parseInt(value.trim()); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("Expected an integer between positions " + start + " and " + end + + ", but got " + value); + } + } + + /** + * Returns the long value from the current to the given position, and moves the current + * position to the next character. + * + * @throws IllegalArgumentException if the string cannot be parsed to a long or end is larger than the string + */ + long longTo(int end) { + int start = position; + String value = stringTo(end); + try { + return Long.parseLong(value.trim()); + } + catch (NumberFormatException e) { + throw new IllegalArgumentException("Expected an integer between positions " + start + " and " + end + + ", but got " + value); + } + } + + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 8334775b8e2..c0c5b0ee0b0 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -356,6 +356,8 @@ public class YqlParser implements Parser { return buildFunctionCall(ast); case LITERAL: return buildLiteral(ast); + case NOT: + return buildNot(ast); default: throw newUnexpectedArgumentException(ast.getOperator(), ExpressionOperator.AND, ExpressionOperator.CALL, @@ -1096,17 +1098,21 @@ public class YqlParser implements Parser { AndItem andItem = new AndItem(); NotItem notItem = new NotItem(); convertVarArgsAnd(ast, 0, andItem, notItem); - Preconditions - .checkArgument(andItem.getItemCount() > 0, - "Vespa does not support AND with no logically positive branches."); if (notItem.getItemCount() == 0) { return andItem; } if (andItem.getItemCount() == 1) { notItem.setPositiveItem(andItem.getItem(0)); - } else { + } else if (andItem.getItemCount() > 1) { notItem.setPositiveItem(andItem); - } + } // else no positives, which is ok + return notItem; + } + + /** Build a "pure" not, without any positive terms. */ + private CompositeItem buildNot(OperatorNode<ExpressionOperator> ast) { + NotItem notItem = new NotItem(); + notItem.addNegativeItem(convertExpression(ast.getArgument(0))); return notItem; } @@ -1663,7 +1669,7 @@ public class YqlParser implements Parser { "Expected operator READ_FIELD or PRPPREF, got %s.", ast.getOperator()); } - private static void addItems(OperatorNode<ExpressionOperator> ast, WeightedSetItem out) { + private void addItems(OperatorNode<ExpressionOperator> ast, WeightedSetItem out) { switch (ast.getOperator()) { case MAP: addStringItems(ast, out); @@ -1671,6 +1677,10 @@ public class YqlParser implements Parser { case ARRAY: addLongItems(ast, out); break; + case VARREF: + Preconditions.checkState(userQuery != null, "Query properties are not available"); + ParameterListParser.addItemsFromString(userQuery.properties().getString(ast.getArgument(0, String.class)), out); + break; default: throw newUnexpectedArgumentException(ast.getOperator(), ExpressionOperator.ARRAY, ExpressionOperator.MAP); @@ -1698,10 +1708,8 @@ public class YqlParser implements Parser { OperatorNode<ExpressionOperator> tokenValueNode = args.get(0); assertHasOperator(tokenValueNode, ExpressionOperator.LITERAL); Number tokenValue = tokenValueNode.getArgument(0, Number.class); - Preconditions.checkArgument(tokenValue instanceof Integer - || tokenValue instanceof Long, - "Expected Integer or Long, got %s.", tokenValue.getClass() - .getName()); + Preconditions.checkArgument(tokenValue instanceof Integer || tokenValue instanceof Long, + "Expected Integer or Long, got %s.", tokenValue.getClass().getName()); OperatorNode<ExpressionOperator> tokenWeightNode = args.get(1); assertHasOperator(tokenWeightNode, ExpressionOperator.LITERAL); diff --git a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj index d79f78ef896..46117374e59 100644 --- a/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj +++ b/container-search/src/main/javacc/com/yahoo/prelude/semantics/parser/SemanticsParser.jj @@ -6,7 +6,6 @@ options { CACHE_TOKENS = true; DEBUG_PARSER = false; ERROR_REPORTING = true; - STATIC = false; UNICODE_INPUT = true; } @@ -15,12 +14,23 @@ PARSER_BEGIN(SemanticsParser) package com.yahoo.prelude.semantics.parser; import com.yahoo.javacc.UnicodeUtilities; +import com.yahoo.language.process.StemMode; +import com.yahoo.language.Linguistics; +import com.yahoo.language.Language; import com.yahoo.prelude.semantics.*; import com.yahoo.prelude.semantics.rule.*; +import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.prelude.query.TermType; public class SemanticsParser { + private RuleBaseLinguistics linguistics; + + public SemanticsParser(java.io.Reader stream, Linguistics linguistics) { + this(stream); + this.linguistics = new RuleBaseLinguistics(linguistics); + } + } PARSER_END(SemanticsParser) @@ -77,6 +87,7 @@ TOKEN : <SMALLER: "<"> | <SMALLEREQUALS: "<="> | <STEMMINGDIRECTIVE: "@stemming"> | + <LANGUAGEDIRECTIVE: "@language"> | <SUPERDIRECTIVE: "@super"> | <IDENTIFIER: (~[ "\u0000"-"\u002f","\u003a"-"\u003f","\u005b"-"\u005d","\u007b"-"\u00a7","\u00a9","\u00ab"-"\u00ae","\u00b0"-"\u00b3","\u00b6"-"\u00b7","\u00b9","\u00bb"-"\u00bf", @@ -114,16 +125,20 @@ RuleBase semanticRules(RuleBase rules,RuleImporter importer) : // ---------------------------------- Directive --------------------------------------- -RuleBase directive(RuleBase rules,RuleImporter importer) : +RuleBase directive(RuleBase rules, RuleImporter importer) : { String name; } { - ( includeDirective(rules,importer) | defaultDirective(rules) | automataDirective(rules,importer) | stemmingDirective(rules) ) + ( includeDirective(rules, importer) | + defaultDirective(rules) | + automataDirective(rules, importer) | + stemmingDirective(rules) | + languageDirective(rules) ) { return rules; } } -void includeDirective(RuleBase rules,RuleImporter importer) : +void includeDirective(RuleBase rules, RuleImporter importer) : { String name; } @@ -131,25 +146,24 @@ void includeDirective(RuleBase rules,RuleImporter importer) : <INCLUDEDIRECTIVE> <LEFTBRACE> name=stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)? { try { - importer.include(name,rules); + importer.include(name, rules); } catch (java.io.IOException e) { - ParseException ep=new ParseException("Could not read included rule base '" + - name + "'"); + ParseException ep=new ParseException("Could not read included rule base '" + name + "'"); ep.initCause(e); throw ep; } } } -void automataDirective(RuleBase rules,RuleImporter importer) : +void automataDirective(RuleBase rules, RuleImporter importer) : { String name; } { - <AUTOMATADIRECTIVE> <LEFTBRACE> name=stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)? + <AUTOMATADIRECTIVE> <LEFTBRACE> name = stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)? { - importer.setAutomata(rules,name); + importer.setAutomata(rules, name); } } @@ -168,9 +182,20 @@ void stemmingDirective(RuleBase rules) : String booleanString; } { - <STEMMINGDIRECTIVE> <LEFTBRACE> booleanString=stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)? + <STEMMINGDIRECTIVE> <LEFTBRACE> booleanString = stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)? + { + linguistics = linguistics.withStemMode(Boolean.parseBoolean(booleanString) ? StemMode.BEST : StemMode.NONE); + } +} + +void languageDirective(RuleBase rules) : +{ + String languageString; +} +{ + <LANGUAGEDIRECTIVE> <LEFTBRACE> languageString = stringOrLiteral() <RIGHTBRACE> (<SEMICOLON>)? { - rules.setStemming(Boolean.parseBoolean(booleanString)); + linguistics = linguistics.withLanguage(Language.from(languageString)); } } @@ -183,10 +208,10 @@ void productionRule(RuleBase rules) : ProductionList production=null; } { - condition=topLevelCondition() rule=productionRuleType() ( production=productionList() )? <SEMICOLON> + condition = topLevelCondition() rule = productionRuleType() ( production = productionList() )? <SEMICOLON> { rule.setCondition(condition); - if (production!=null) rule.setProduction(production); + if (production != null) rule.setProduction(production); rules.addRule(rule); } } @@ -201,16 +226,16 @@ ProductionRule productionRuleType() : ProductionList productionList() : { - ProductionList productionList=new ProductionList(); + ProductionList productionList = new ProductionList(); Production production; int weight=100; } { - ( production=production() (<EXCLAMATION> weight=number())? + ( production = production() (<EXCLAMATION> weight = number())? { production.setWeight(weight); productionList.addProduction(production); - weight=100; + weight = 100; } (<NL>)* ) + { return productionList; } @@ -221,7 +246,7 @@ Production production() : Production production; } { - ( LOOKAHEAD(2) production=namespaceProduction() | production=termProduction() ) + ( LOOKAHEAD(2) production = namespaceProduction() | production = termProduction() ) { return production; } } @@ -229,12 +254,12 @@ TermProduction termProduction() : { TermProduction termProduction; TermType termType; - String label=null; + String label = null; } { - termType=termType() - ( LOOKAHEAD(2) label=label() )? - ( termProduction=nonphraseTermProduction() | termProduction=phraseProduction() ) + termType = termType() + ( LOOKAHEAD(2) label = label() )? + ( termProduction = nonphraseTermProduction() | termProduction = phraseProduction() ) { termProduction.setLabel(label); @@ -248,8 +273,8 @@ TermProduction nonphraseTermProduction() : TermProduction termProduction; } { - ( termProduction=referenceTermProduction() | - termProduction=literalTermProduction() ) + ( termProduction = referenceTermProduction() | + termProduction = literalTermProduction() ) { return termProduction; } @@ -257,14 +282,14 @@ TermProduction nonphraseTermProduction() : LiteralPhraseProduction phraseProduction() : { - LiteralPhraseProduction phraseProduction=new LiteralPhraseProduction(); - String term=null; + LiteralPhraseProduction phraseProduction = new LiteralPhraseProduction(); + String term = null; } { <QUOTE> ( - term=identifier() + term = identifier() { phraseProduction.addTerm(term); } )+ <QUOTE> @@ -277,11 +302,11 @@ NamespaceProduction namespaceProduction() : { String namespace; String key; - String value=null; + String value = null; } { - namespace=identifier() <DOT> key=stringOrLiteral() <EQUALS> value=identifierOrLiteral() - { return new NamespaceProduction(namespace,key,value); } + namespace = identifier() <DOT> key = stringOrLiteral() <EQUALS> value = identifierOrLiteral() + { return new NamespaceProduction(namespace, key, value); } } ReferenceTermProduction referenceTermProduction() : @@ -289,7 +314,7 @@ ReferenceTermProduction referenceTermProduction() : String reference; } { - <LEFTSQUAREBRACKET> reference=referenceIdentifier() <RIGHTSQUAREBRACKET> + <LEFTSQUAREBRACKET> reference = referenceIdentifier() <RIGHTSQUAREBRACKET> { return new ReferenceTermProduction(reference); } } @@ -298,7 +323,7 @@ LiteralTermProduction literalTermProduction() : String literal; } { - literal=identifier() + literal = identifier() { return new LiteralTermProduction(literal); } } @@ -319,7 +344,7 @@ String referenceIdentifier() : String reference; } { - ( reference=identifier() { return reference; } ) + ( reference = identifier() { return reference; } ) | ( <ELLIPSIS> { return "..."; } ) } @@ -332,25 +357,25 @@ void namedCondition(RuleBase rules) : Condition condition; } { - <LEFTSQUAREBRACKET> conditionName=identifier() <RIGHTSQUAREBRACKET> <CONDITION> condition=topLevelCondition() <SEMICOLON> - { rules.addCondition(new NamedCondition(conditionName,condition)); } + <LEFTSQUAREBRACKET> conditionName = identifier() <RIGHTSQUAREBRACKET> <CONDITION> condition = topLevelCondition() <SEMICOLON> + { rules.addCondition(new NamedCondition(conditionName, condition)); } } Condition topLevelCondition() : { Condition condition; - boolean startAnchor=false; - boolean endAnchor=false; + boolean startAnchor = false; + boolean endAnchor = false; } { - ( <DOT> { startAnchor=true; } )? + ( <DOT> { startAnchor = true; } )? ( - LOOKAHEAD(3) condition=choiceCondition() | - LOOKAHEAD(3) condition=sequenceCondition() + LOOKAHEAD(3) condition = choiceCondition() | + LOOKAHEAD(3) condition = sequenceCondition() ) - ( LOOKAHEAD(2) <DOT> { endAnchor=true; } )? + ( LOOKAHEAD(2) <DOT> { endAnchor = true; } )? { - condition.setAnchor(Condition.Anchor.create(startAnchor,endAnchor)); + condition.setAnchor(Condition.Anchor.create(startAnchor, endAnchor)); return condition; } } @@ -361,8 +386,8 @@ Condition condition() : } { ( - ( LOOKAHEAD(3) condition=choiceCondition() - | condition=terminalCondition() ) + ( LOOKAHEAD(3) condition = choiceCondition() + | condition = terminalCondition() ) { return condition; } @@ -374,8 +399,8 @@ Condition terminalOrSequenceCondition() : Condition condition; } { - ( LOOKAHEAD(3) condition=sequenceCondition() | - condition=terminalCondition() ) + ( LOOKAHEAD(3) condition = sequenceCondition() | + condition = terminalCondition() ) { return condition; } } @@ -384,20 +409,20 @@ Condition terminalCondition() : Condition condition; } { - ( condition=notCondition() | condition=terminalOrComparisonCondition() ) + ( condition = notCondition() | condition = terminalOrComparisonCondition() ) { return condition; } } Condition terminalOrComparisonCondition() : { - Condition condition,rightCondition; + Condition condition, rightCondition; String comparison; } { - condition=reallyTerminalCondition() - ( comparison=comparison() ( LOOKAHEAD(2) rightCondition=nestedCondition() | rightCondition=reallyTerminalCondition() ) -// ( comparison=comparison() rightCondition=condition() - { condition=new ComparisonCondition(condition,comparison,rightCondition); } + condition = reallyTerminalCondition() + ( comparison = comparison() ( LOOKAHEAD(2) rightCondition = nestedCondition() | rightCondition = reallyTerminalCondition() ) +// ( comparison = comparison() rightCondition = condition() + { condition = new ComparisonCondition(condition, comparison, rightCondition); } ) ? { return condition; } @@ -405,10 +430,10 @@ Condition terminalOrComparisonCondition() : Condition reallyTerminalCondition() : { - String label=null; - String context=null; - String nameSpace=null; - Condition condition=null; + String label = null; + String context = null; + String nameSpace = null; + Condition condition = null; } { // This body looks like this to distinguish these two cases @@ -416,20 +441,20 @@ Condition reallyTerminalCondition() : // condition . (end anchor) ( LOOKAHEAD(8) ( - ( LOOKAHEAD(2) context=context() )? - ( nameSpace=nameSpace() ) - ( LOOKAHEAD(2) label=label() )? - condition=terminalConditionBody() + ( LOOKAHEAD(2) context = context() )? + ( nameSpace = nameSpace() ) + ( LOOKAHEAD(2) label = label() )? + condition = terminalConditionBody() ) | ( - ( LOOKAHEAD(2) context=context() )? - ( LOOKAHEAD(2) label=label() )? - condition=terminalConditionBody() + ( LOOKAHEAD(2) context = context() )? + ( LOOKAHEAD(2) label = label() )? + condition = terminalConditionBody() ) ) { - if (context!=null) + if (context != null) condition.setContextName(context); condition.setLabel(label); condition.setNameSpace(nameSpace); @@ -440,18 +465,18 @@ Condition reallyTerminalCondition() : Condition terminalConditionBody() : { - Condition condition=null; + Condition condition = null; } { ( - LOOKAHEAD(2) condition=conditionReference() | - condition=termCondition() | - condition=nestedCondition() | - condition=nonReferableEllipsisCondition() | - condition=referableEllipsisCondition() | - condition=superCondition() | - condition=literalCondition() | - condition=compositeItemCondition()) + LOOKAHEAD(2) condition = conditionReference() | + condition = termCondition() | + condition = nestedCondition() | + condition = nonReferableEllipsisCondition() | + condition = referableEllipsisCondition() | + condition = superCondition() | + condition = literalCondition() | + condition = compositeItemCondition()) { return condition; } } @@ -460,7 +485,7 @@ Condition notCondition() : Condition condition; } { - <EXCLAMATION> condition=terminalOrComparisonCondition() + <EXCLAMATION> condition = terminalOrComparisonCondition() { return new NotCondition(condition); } } @@ -470,7 +495,7 @@ ConditionReference conditionReference() : String conditionName; } { - <LEFTSQUAREBRACKET> conditionName=identifier() <RIGHTSQUAREBRACKET> + <LEFTSQUAREBRACKET> conditionName = identifier() <RIGHTSQUAREBRACKET> { return new ConditionReference(conditionName); } } @@ -494,23 +519,23 @@ Condition nestedCondition() : Condition condition; } { - <LEFTBRACE> condition=choiceCondition() <RIGHTBRACE> + <LEFTBRACE> condition = choiceCondition() <RIGHTBRACE> { return condition; } } Condition sequenceCondition() : { - SequenceCondition sequenceCondition=new SequenceCondition(); + SequenceCondition sequenceCondition = new SequenceCondition(); Condition condition; } { - condition=terminalCondition() + condition = terminalCondition() { sequenceCondition.addCondition(condition); } - ( LOOKAHEAD(2) condition=terminalCondition() + ( LOOKAHEAD(2) condition = terminalCondition() { sequenceCondition.addCondition(condition); } )* { - if (sequenceCondition.conditionSize()==1) + if (sequenceCondition.conditionSize() == 1) return sequenceCondition.removeCondition(0); else return sequenceCondition; @@ -519,17 +544,17 @@ Condition sequenceCondition() : Condition choiceCondition() : { - ChoiceCondition choiceCondition=new ChoiceCondition(); + ChoiceCondition choiceCondition = new ChoiceCondition(); Condition condition; } { - condition=terminalOrSequenceCondition() + condition = terminalOrSequenceCondition() { choiceCondition.addCondition(condition); } - ( LOOKAHEAD(3) (<NL>)* <COMMA> (<NL>)* condition=terminalOrSequenceCondition() + ( LOOKAHEAD(3) (<NL>)* <COMMA> (<NL>)* condition = terminalOrSequenceCondition() { choiceCondition.addCondition(condition); } ) * { - if (choiceCondition.conditionSize()==1) + if (choiceCondition.conditionSize() == 1) return choiceCondition.removeCondition(0); else return choiceCondition; @@ -542,7 +567,7 @@ TermCondition termCondition() : } { ( str = identifier() ) - { return new TermCondition(str); } + { return new TermCondition(str, linguistics); } } SuperCondition superCondition() : { } @@ -566,7 +591,7 @@ CompositeItemCondition compositeItemCondition() : CompositeItemCondition compositeItemCondition = new CompositeItemCondition(); } { - ( <QUOTE> ( condition=terminalConditionBody() { compositeItemCondition.addCondition(condition); } ) <QUOTE> ) + ( <QUOTE> ( condition = terminalConditionBody() { compositeItemCondition.addCondition(condition); } ) <QUOTE> ) { return compositeItemCondition; } } diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java index 1d2f92063fe..11424cc7e4e 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/QueryCanonicalizerTestCase.java @@ -291,22 +291,22 @@ public class QueryCanonicalizerTestCase { } @Test - public void testNegativeMustHavePositive() { + public void testNegative() { NotItem root = new NotItem(); root.addNegativeItem(new WordItem("negative")); - assertCanonicalized("+(null) -negative","Can not search for only negative items", root); + assertCanonicalized("-negative",null, root); } @Test - public void testNegativeMustHavePositiveNested() { + public void testNegativeOnly() { CompositeItem root = new AndItem(); NotItem not = new NotItem(); root.addItem(not); root.addItem(new WordItem("word")); not.addNegativeItem(new WordItem("negative")); - assertCanonicalized("AND (+(null) -negative) word","Can not search for only negative items", root); + assertCanonicalized("AND (-negative) word",null, root); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java index ca5bb4d4cd2..bf99a709df3 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/parser/test/SemanticsParserTestCase.java @@ -4,6 +4,7 @@ package com.yahoo.prelude.semantics.parser.test; import java.util.Iterator; import com.yahoo.javacc.UnicodeUtilities; +import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleImporter; import com.yahoo.prelude.semantics.parser.ParseException; @@ -24,8 +25,8 @@ public class SemanticsParserTestCase { @Test public void testRuleReading() throws java.io.IOException, ParseException { - RuleBase rules=new RuleImporter().importFile(ROOT + "rules.sr"); - Iterator<?> i=rules.ruleIterator(); + RuleBase rules = new RuleImporter(new SimpleLinguistics()).importFile(ROOT + "rules.sr"); + Iterator<?> i = rules.ruleIterator(); assertEquals("[listing] [preposition] [place] -> listing:[listing] place:[place]!150", i.next().toString()); assertEquals("[listing] [place] +> place:[place]", @@ -36,10 +37,10 @@ public class SemanticsParserTestCase { i.next().toString()); assertEquals("digital camera -> digicamera", i.next().toString()); - assertEquals("(parameter.ranking='cat'), (parameter.ranking='cat0') -> one",i.next().toString()); + assertEquals("(parameter.ranking='cat'), (parameter.ranking='cat0') -> one", i.next().toString()); assertFalse(i.hasNext()); - i=rules.conditionIterator(); + i = rules.conditionIterator(); assertEquals("[listing] :- restaurant, shop, cafe, hotel", i.next().toString()); assertEquals("[preposition] :- in, at, near", @@ -53,7 +54,7 @@ public class SemanticsParserTestCase { assertFalse(i.hasNext()); assertTrue(rules.isDefault()); - assertEquals(ROOT + "semantics.fsa",rules.getAutomataFile()); + assertEquals(ROOT + "semantics.fsa", rules.getAutomataFile()); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java index ac1791ae91a..394752f8aa1 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/BacktrackingTestCase.java @@ -30,7 +30,7 @@ public class BacktrackingTestCase { static { try { - searcher = new SemanticSearcher(new RuleImporter().importFile(root + "backtrackingrules.sr")); + searcher = new SemanticSearcher(new RuleImporter(new SimpleLinguistics()).importFile(root + "backtrackingrules.sr")); } catch (Exception e) { throw new RuntimeException(e); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java index 86ee9b5948b..eb69372c22b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConditionTestCase.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.test; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.search.Query; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.engine.Evaluation; @@ -24,15 +26,17 @@ public class ConditionTestCase { @Test public void testTermCondition() { - TermCondition term=new TermCondition("foo"); - Query query=new Query("?query=foo"); + var linguistics = new RuleBaseLinguistics(new SimpleLinguistics()); + TermCondition term = new TermCondition("foo", linguistics); + Query query = new Query("?query=foo"); assertTrue(term.matches(new Evaluation(query).freshRuleEvaluation())); } @Test public void testSequenceCondition() { - TermCondition term1 = new TermCondition("foo"); - TermCondition term2 = new TermCondition("bar"); + var linguistics = new RuleBaseLinguistics(new SimpleLinguistics()); + TermCondition term1 = new TermCondition("foo", linguistics); + TermCondition term2 = new TermCondition("bar",linguistics); SequenceCondition sequence = new SequenceCondition(); sequence.addCondition(term1); sequence.addCondition(term2); @@ -46,8 +50,9 @@ public class ConditionTestCase { @Test public void testChoiceCondition() { - TermCondition term1 = new TermCondition("foo"); - TermCondition term2 = new TermCondition("bar"); + var linguistics = new RuleBaseLinguistics(new SimpleLinguistics()); + TermCondition term1 = new TermCondition("foo", linguistics); + TermCondition term2 = new TermCondition("bar", linguistics); ChoiceCondition choice = new ChoiceCondition(); choice.addCondition(term1); choice.addCondition(term2); @@ -61,7 +66,8 @@ public class ConditionTestCase { @Test public void testNamedConditionReference() { - TermCondition term = new TermCondition("foo"); + var linguistics = new RuleBaseLinguistics(new SimpleLinguistics()); + TermCondition term = new TermCondition("foo", linguistics); NamedCondition named = new NamedCondition("cond",term); ConditionReference reference = new ConditionReference("cond"); @@ -69,8 +75,7 @@ public class ConditionTestCase { ProductionRule rule = new ReplacingProductionRule(); rule.setCondition(reference); rule.setProduction(new ProductionList()); - RuleBase ruleBase = new RuleBase(); - ruleBase.setName("test"); + RuleBase ruleBase = new RuleBase("test", linguistics.linguistics()); ruleBase.addCondition(named); ruleBase.addRule(rule); ruleBase.initialize(); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java index 80c9e898302..6d5b9459833 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ConfigurationTestCase.java @@ -37,7 +37,7 @@ public class ConfigurationTestCase { static { semanticRulesConfig = new ConfigGetter<>(SemanticRulesConfig.class).getConfig("file:" + root + "semantic-rules.cfg"); - searcher=new SemanticSearcher(semanticRulesConfig); + searcher = new SemanticSearcher(semanticRulesConfig, new SimpleLinguistics()); } protected void assertSemantics(String result, String input, String baseName) { @@ -54,46 +54,46 @@ public class ConfigurationTestCase { @Test public void testReadingConfigurationRuleBase() { - RuleBase parent=searcher.getRuleBase("parent"); + RuleBase parent = searcher.getRuleBase("parent"); assertNotNull(parent); - assertEquals("parent",parent.getName()); - assertEquals("semantic-rules.cfg",parent.getSource()); + assertEquals("parent", parent.getName()); + assertEquals("semantic-rules.cfg", parent.getSource()); } @Test - public void testParent() throws Exception { - assertSemantics("vehiclebrand:audi","audi cars","parent"); - assertSemantics("vehiclebrand:alfa","alfa bus","parent"); - assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","parent.sr"); - assertSemantics("AND vw car", "vw cars","parent"); - assertSemantics("AND skoda car", "skoda cars","parent.sr"); + public void testParent() { + assertSemantics("vehiclebrand:audi", "audi cars", "parent"); + assertSemantics("vehiclebrand:alfa", "alfa bus", "parent"); + assertSemantics("AND vehiclebrand:bmw expensivetv", "bmw motorcycle", "parent.sr"); + assertSemantics("AND vw car", "vw cars", "parent"); + assertSemantics("AND skoda car", "skoda cars", "parent.sr"); } @Test - public void testChild1() throws Exception { - assertSemantics("vehiclebrand:skoda","audi cars","child1.sr"); - assertSemantics("vehiclebrand:alfa", "alfa bus","child1"); - assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","child1"); - assertSemantics("vehiclebrand:skoda","vw cars","child1"); - assertSemantics("AND skoda car", "skoda cars","child1"); + public void testChild1() { + assertSemantics("vehiclebrand:skoda", "audi cars", "child1.sr"); + assertSemantics("vehiclebrand:alfa", "alfa bus", "child1"); + assertSemantics("AND vehiclebrand:bmw expensivetv", "bmw motorcycle", "child1"); + assertSemantics("vehiclebrand:skoda", "vw cars", "child1"); + assertSemantics("AND skoda car", "skoda cars", "child1"); } @Test - public void testChild2() throws Exception { - assertSemantics("vehiclebrand:audi","audi cars","child2"); - assertSemantics("vehiclebrand:alfa","alfa bus","child2.sr"); - assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","child2.sr"); - assertSemantics("AND vw car","vw cars","child2"); - assertSemantics("vehiclebrand:skoda","skoda cars","child2"); + public void testChild2() { + assertSemantics("vehiclebrand:audi", "audi cars", "child2"); + assertSemantics("vehiclebrand:alfa", "alfa bus", "child2.sr"); + assertSemantics("AND vehiclebrand:bmw expensivetv", "bmw motorcycle", "child2.sr"); + assertSemantics("AND vw car", "vw cars", "child2"); + assertSemantics("vehiclebrand:skoda", "skoda cars", "child2"); } @Test - public void testGrandchild() throws Exception { - assertSemantics("vehiclebrand:skoda","audi cars","grandchild.sr"); - assertSemantics("vehiclebrand:alfa","alfa bus","grandchild"); - assertSemantics("AND vehiclebrand:bmw expensivetv","bmw motorcycle","grandchild"); - assertSemantics("vehiclebrand:skoda","vw cars","grandchild"); - assertSemantics("vehiclebrand:skoda","skoda cars","grandchild"); + public void testGrandchild() { + assertSemantics("vehiclebrand:skoda", "audi cars", "grandchild.sr"); + assertSemantics("vehiclebrand:alfa", "alfa bus", "grandchild"); + assertSemantics("AND vehiclebrand:bmw expensivetv", "bmw motorcycle", "grandchild"); + assertSemantics("vehiclebrand:skoda", "vw cars", "grandchild"); + assertSemantics("vehiclebrand:skoda", "skoda cars", "grandchild"); } @Test diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java index fb86beaa9bc..76c8c3966b7 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/DuplicateRuleTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.test; +import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.prelude.semantics.RuleBaseException; import com.yahoo.prelude.semantics.RuleImporter; import com.yahoo.prelude.semantics.parser.ParseException; @@ -14,18 +15,18 @@ import static org.junit.Assert.fail; */ public class DuplicateRuleTestCase { - private final String root="src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; + private final String root = "src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; @Test public void testDuplicateRuleBaseLoading() throws java.io.IOException, ParseException { if (System.currentTimeMillis() > 0) return; // TODO: Include this test... try { - new RuleImporter().importFile(root + "rules.sr"); + new RuleImporter(new SimpleLinguistics()).importFile(root + "rules.sr"); fail("Did not detect duplicate condition names"); } catch (RuleBaseException e) { - assertEquals("Duplicate condition 'something' in 'duplicaterules.sr'",e.getMessage()); + assertEquals("Duplicate condition 'something' in 'duplicaterules.sr'", e.getMessage()); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExactMatchTrickTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExactMatchTrickTestCase.java index a2ff6ae5b82..c566b05405d 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExactMatchTrickTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ExactMatchTrickTestCase.java @@ -21,13 +21,13 @@ public class ExactMatchTrickTestCase extends RuleBaseAbstractTestCase { @Test public void testCompleteMatchWithNegative() { // Notice ordering bug - assertSemantics("+(AND default:primetime default:in default:time default:no) -regionexcl:us", + assertSemantics("+(AND default:primetime default:in default:time default:no TRUE) -regionexcl:us", new Query(QueryTestCase.httpEncode("?query=primetime ANDNOT regionexcl:us&type=adv"))); } @Test public void testCompleteMatchWithFilterAndNegative() { - assertSemantics("AND (+(AND default:primetime default:in default:time default:no) -regionexcl:us) |lang:en", + assertSemantics("AND (+(AND default:primetime default:in default:time default:no TRUE) -regionexcl:us) |lang:en", new Query(QueryTestCase.httpEncode("?query=primetime ANDNOT regionexcl:us&type=adv&filter=+lang:en"))); } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java index d93fd218259..e9364074281 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/InheritanceTestCase.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.StringTokenizer; import com.yahoo.component.chain.Chain; +import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.search.Query; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleBaseException; @@ -24,7 +25,6 @@ import static org.junit.Assert.fail; /** * @author bratseth */ -@SuppressWarnings("deprecation") public class InheritanceTestCase { private static final String root = "src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; @@ -34,10 +34,10 @@ public class InheritanceTestCase { static { try { - parent = RuleBase.createFromFile(root + "inheritingrules/parent.sr", null); - child1 = RuleBase.createFromFile(root + "inheritingrules/child1.sr", null); - child2 = RuleBase.createFromFile(root + "inheritingrules/child2.sr", null); - grandchild = RuleBase.createFromFile(root + "inheritingrules/grandchild.sr", null); + parent = RuleBase.createFromFile(root + "inheritingrules/parent.sr", null, new SimpleLinguistics()); + child1 = RuleBase.createFromFile(root + "inheritingrules/child1.sr", null, new SimpleLinguistics()); + child2 = RuleBase.createFromFile(root + "inheritingrules/child2.sr", null, new SimpleLinguistics()); + grandchild = RuleBase.createFromFile(root + "inheritingrules/grandchild.sr", null, new SimpleLinguistics()); grandchild.setDefault(true); searcher = new SemanticSearcher(parent, child1, child2, grandchild); @@ -77,7 +77,7 @@ public class InheritanceTestCase { public void testInclusionOrderAndContentDump() { StringTokenizer lines = new StringTokenizer(grandchild.toContentString(),"\n",false); assertEquals("vw -> audi", lines.nextToken()); - assertEquals("cars -> car", lines.nextToken()); + assertEquals("car -> car", lines.nextToken()); assertEquals("[brand] [vehicle] -> vehiclebrand:[brand]", lines.nextToken()); assertEquals("vehiclebrand:bmw +> expensivetv", lines.nextToken()); assertEquals("vehiclebrand:audi -> vehiclebrand:skoda", lines.nextToken()); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/MusicTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MusicTestCase.java new file mode 100644 index 00000000000..006dcb3c714 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/MusicTestCase.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import org.junit.Test; + +/** + * Tests the rewriting in the semanticsearcher system test + * + * @author bratseth + */ +public class MusicTestCase { + + @Test + public void testMusic() { + var tester = new RuleBaseTester("music.sr"); + tester.assertSemantics("AND song:together artist:youngbloods", "together by youngbloods"); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/NoStemmingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/NoStemmingTestCase.java index 7acecdcf00b..fbdd72fe6ac 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/NoStemmingTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/NoStemmingTestCase.java @@ -17,13 +17,13 @@ public class NoStemmingTestCase extends RuleBaseAbstractTestCase { /** Should rewrite correctly */ @Test public void testCorrectRewriting1() { - assertSemantics("+(AND i:arts i:sciences) -i:b","i:as -i:b"); + assertSemantics("+(AND i:arts i:sciences TRUE) -i:b","i:as -i:b"); } /** Should rewrite correctly too */ @Test public void testCorrectRewriting2() { - assertSemantics("+(AND i:arts i:sciences i:crafts) -i:b","i:asc -i:b"); + assertSemantics("+(AND i:arts i:sciences i:crafts TRUE) -i:b","i:asc -i:b"); } /** Should not rewrite */ diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java index cd5743c6d77..376da065f4d 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ParameterTestCase.java @@ -20,66 +20,65 @@ public class ParameterTestCase extends RuleBaseAbstractTestCase { /** Tests parameter literal matching */ @Test public void testLiteralEquals() { - assertSemantics("a","a"); - assertSemantics("RANK a foo:a","a&ranking=category"); - assertSemantics("a","a&ranking=somethingelse"); - assertSemantics("a","a&otherparam=category"); + assertSemantics("a", "a"); + assertSemantics("RANK a foo:a", "a&ranking=category"); + assertSemantics("a", "a&ranking=somethingelse"); + assertSemantics("a", "a&otherparam=category"); } /** Tests parameter matching of larger */ @Test public void testLarger() { - assertSemantics("a","a"); - assertSemantics("AND a largepage","a&hits=11"); - assertSemantics("AND a largepage","a&hits=12"); + assertSemantics("a", "a"); + assertSemantics("AND a largepage", "a&hits=11"); + assertSemantics("AND a largepage", "a&hits=12"); } /** Tests parameter containment matching */ @Test public void testContainsAsList() { assertSemantics("a","a"); - assertSemantics("AND a intent:music","a&search=music"); - assertSemantics("AND a intent:music","a&search=music,books"); - assertSemantics("AND a intent:music","a&search=kanoos,music,books"); + assertSemantics("AND a intent:music", "a&search=music"); + assertSemantics("AND a intent:music", "a&search=music,books"); + assertSemantics("AND a intent:music", "a&search=kanoos,music,books"); } /** Tests parameter production */ @Test public void testParameterProduction() { - assertParameterSemantics("AND a b c","a b c","search","[letters, alphabet]"); - assertParameterSemantics("AND a c d","a c d","search","[letters, someletters]"); - assertParameterSemantics("+(AND a d e) -letter:c","a d e","search","[someletters]"); - assertParameterSemantics("AND a d f","a d f","rank-profile","foo"); - assertParameterSemantics("AND a f g","a f g","grouping.nolearning","true"); + assertParameterSemantics("AND a b c", "a b c", "search", "[letters, alphabet]"); + assertParameterSemantics("AND a c d", "a c d", "search", "[letters, someletters]"); + assertParameterSemantics("+(AND a d e) -letter:c", "a d e", "search", "[someletters]"); + assertParameterSemantics("AND a d f", "a d f", "rank-profile", "foo"); + assertParameterSemantics("AND a f g", "a f g", "grouping.nolearning", "true"); } @Test public void testMultipleAlternativeParameterValuesInCondition() { - assertInputRankParameterSemantics("one","foo","cat"); - assertInputRankParameterSemantics("one","foo","cat0"); - assertInputRankParameterSemantics("one","bar","cat"); - assertInputRankParameterSemantics("one","bar","cat0"); - assertInputRankParameterSemantics("AND one one","foo+bar","cat0"); - assertInputRankParameterSemantics("AND fuki sushi","fuki+sushi","cat0"); + assertInputRankParameterSemantics("one", "foo", "cat"); + assertInputRankParameterSemantics("one", "foo", "cat0"); + assertInputRankParameterSemantics("one", "bar", "cat"); + assertInputRankParameterSemantics("one", "bar", "cat0"); + assertInputRankParameterSemantics("AND one one", "foo+bar", "cat0"); + assertInputRankParameterSemantics("AND fuki sushi", "fuki+sushi", "cat0"); } - private void assertInputRankParameterSemantics(String producedQuery,String inputQuery, - String rankParameterValue) { - assertInputRankParameterSemantics(producedQuery,inputQuery,rankParameterValue,0); + private void assertInputRankParameterSemantics(String producedQuery,String inputQuery, String rankParameterValue) { + assertInputRankParameterSemantics(producedQuery, inputQuery, rankParameterValue, 0); } - private void assertInputRankParameterSemantics(String producedQuery,String inputQuery, - String rankParameterValue,int tracelevel) { - Query query=new Query("?query=" + inputQuery + "&tracelevel=0&tracelevel.rules=" + tracelevel); + private void assertInputRankParameterSemantics(String producedQuery, String inputQuery, + String rankParameterValue, int tracelevel) { + Query query = new Query("?query=" + inputQuery + "&tracelevel=0&tracelevel.rules=" + tracelevel); query.getRanking().setProfile(rankParameterValue); query.properties().set("tracelevel.rules", tracelevel); assertSemantics(producedQuery, query); } - private void assertParameterSemantics(String producedQuery,String inputQuery, - String producedParameterName,String producedParameterValue) { - Query query=assertSemantics(producedQuery,inputQuery); - assertEquals(producedParameterValue,query.properties().getString(producedParameterName)); + private void assertParameterSemantics(String producedQuery, String inputQuery, + String producedParameterName, String producedParameterValue) { + Query query = assertSemantics(producedQuery, inputQuery); + assertEquals(producedParameterValue, query.properties().getString(producedParameterName)); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java index 8b883759215..b91e9441a2b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/ProductionRuleTestCase.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.semantics.test; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.semantics.engine.RuleBaseLinguistics; import com.yahoo.search.Query; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.engine.Evaluation; @@ -25,7 +27,8 @@ public class ProductionRuleTestCase { @Test public void testProductionRule() { - TermCondition term = new TermCondition("sony"); + var linguistics = new RuleBaseLinguistics(new SimpleLinguistics()); + TermCondition term = new TermCondition("sony", linguistics); NamedCondition named = new NamedCondition("brand", term); ConditionReference reference = new ConditionReference("brand"); @@ -38,8 +41,7 @@ public class ProductionRuleTestCase { rule.setProduction(productionList); // To initialize the condition reference... - RuleBase ruleBase = new RuleBase(); - ruleBase.setName("test"); + RuleBase ruleBase = new RuleBase("test", linguistics.linguistics()); ruleBase.addCondition(named); ruleBase.addRule(rule); ruleBase.initialize(); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java index baccb73cd93..84e47edae29 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseAbstractTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.prelude.semantics.test; import com.yahoo.component.chain.Chain; +import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.search.Query; import com.yahoo.prelude.semantics.RuleBase; import com.yahoo.prelude.semantics.RuleBaseException; @@ -16,7 +17,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; /** - * Tests semantic searching + * DO NOT USE. Use RuleBaseTester instead * * @author bratseth */ @@ -37,7 +38,7 @@ public abstract class RuleBaseAbstractTestCase { try { if (automataFileName != null) automataFileName = root + automataFileName; - RuleBase ruleBase = RuleBase.createFromFile(root + ruleBaseName, automataFileName); + RuleBase ruleBase = RuleBase.createFromFile(root + ruleBaseName, automataFileName, new SimpleLinguistics()); return new SemanticSearcher(ruleBase); } catch (Exception e) { throw new RuleBaseException("Initialization of rule base '" + ruleBaseName + "' failed",e); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseTester.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseTester.java new file mode 100644 index 00000000000..cc9e758a0e0 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/RuleBaseTester.java @@ -0,0 +1,79 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.semantics.test; + +import com.yahoo.component.chain.Chain; +import com.yahoo.language.opennlp.OpenNlpLinguistics; +import com.yahoo.prelude.semantics.RuleBase; +import com.yahoo.prelude.semantics.RuleBaseException; +import com.yahoo.prelude.semantics.SemanticSearcher; +import com.yahoo.search.Query; +import com.yahoo.search.Searcher; +import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.test.QueryTestCase; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +/** + * Helper for testing with a rule base. + * Replace subclassing of RuleBaseAbstractTestCase by this. + * + * @author bratseth + */ +public class RuleBaseTester { + + private final String root = "src/test/java/com/yahoo/prelude/semantics/test/rulebases/"; + private final SemanticSearcher searcher; + + public RuleBaseTester(String ruleBaseName) { + this(ruleBaseName, null); + } + + public RuleBaseTester(String ruleBaseName, String automataFileName) { + searcher = createSearcher(ruleBaseName, automataFileName); + } + + private SemanticSearcher createSearcher(String ruleBaseName,String automataFileName) { + try { + if (automataFileName != null) + automataFileName = root + automataFileName; + RuleBase ruleBase = RuleBase.createFromFile(root + ruleBaseName, automataFileName, new OpenNlpLinguistics()); + return new SemanticSearcher(ruleBase); + } catch (Exception e) { + throw new RuleBaseException("Initialization of rule base '" + ruleBaseName + "' failed", e); + } + } + + public Query assertSemantics(String result, String input) { + return assertSemantics(result, input, 0); + } + + public Query assertSemantics(String result, String input, int tracelevel) { + return assertSemantics(result, input, tracelevel, Query.Type.ALL); + } + + public Query assertSemantics(String result, String input, int tracelevel, Query.Type queryType) { + Query query = new Query("?query=" + QueryTestCase.httpEncode(input) + "&tracelevel=0&tracelevel.rules=" + tracelevel + + "&language=und&type=" + queryType); + return assertSemantics(result, query); + } + + public Query assertSemantics(String result, Query query) { + createExecution(searcher).search(query); + assertEquals(result, query.getModel().getQueryTree().getRoot().toString()); + return query; + } + + private Execution createExecution(Searcher searcher) { + return new Execution(chainedAsSearchChain(searcher), Execution.Context.createContextStub()); + } + + private Chain<Searcher> chainedAsSearchChain(Searcher topOfChain) { + List<Searcher> searchers = new ArrayList<>(); + searchers.add(topOfChain); + return new Chain<>(searchers); + } + +} diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java index 94332a52fed..69faed1da90 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java @@ -54,7 +54,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase { } private static Item parseQuery(String query) { - AllParser parser = new AllParser(new ParserEnvironment().setLinguistics(TestLinguistics.INSTANCE)); + AllParser parser = new AllParser(new ParserEnvironment().setLinguistics(TestLinguistics.INSTANCE), false); return parser.parse(new Parsable().setQuery(query).setLanguage(Language.CHINESE_SIMPLIFIED)).getRoot(); } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java index 29cc5c6e23a..76b2d3991c1 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SemanticSearcherTestCase.java @@ -142,11 +142,6 @@ public class SemanticSearcherTestCase extends RuleBaseAbstractTestCase { } @Test - public void testPluralReplaceBecomesSingular() { - assertSemantics("AND from:paris to:texas","pariss to texass"); - } - - @Test public void testOrProduction() { assertSemantics("OR something somethingelse", "something"); } @@ -155,7 +150,7 @@ public class SemanticSearcherTestCase extends RuleBaseAbstractTestCase { @Test public void testWeightedSetItem() { Query q = new Query(); - WeightedSetItem weightedSet=new WeightedSetItem("fieldName"); + WeightedSetItem weightedSet = new WeightedSetItem("fieldName"); weightedSet.addToken("a", 1); weightedSet.addToken("b", 2); q.getModel().getQueryTree().setRoot(weightedSet); diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java index 136381df552..b8efbf7422b 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/StemmingTestCase.java @@ -4,34 +4,52 @@ package com.yahoo.prelude.semantics.test; import org.junit.Test; /** - * Tests a case reported by tularam + * Tests stemming. * * @author bratseth */ -public class StemmingTestCase extends RuleBaseAbstractTestCase { +public class StemmingTestCase { - public StemmingTestCase() { - super("stemming.sr"); + @Test + public void testRewritingDueToStemmingInQuery() { + var tester = new RuleBaseTester("stemming.sr"); + tester.assertSemantics("+(AND i:vehicle TRUE) -i:s", "i:cars -i:s"); } @Test - public void testRewritingDueToStemmingInQuery() { - assertSemantics("+i:vehicle -i:s","i:cars -i:s"); + public void testNoRewritingDueToStemmingInQueryWhenStemmingDisabled() { + var tester = new RuleBaseTester("stemming-none.sr"); + tester.assertSemantics("+i:cars -i:s", "i:cars -i:s"); } @Test public void testRewritingDueToStemmingInRule() { - assertSemantics("+i:animal -i:s","i:horse -i:s"); + var tester = new RuleBaseTester("stemming.sr"); + tester.assertSemantics("+(AND i:animal TRUE) -i:s", "i:horse -i:s"); + } + + @Test + public void testNoRewritingDueToStemmingInRuleWhenStemmingDisabled() { + var tester = new RuleBaseTester("stemming-none.sr"); + tester.assertSemantics("+i:horse -i:s", "i:horse -i:s"); } @Test public void testRewritingDueToExactMatch() { - assertSemantics("+(AND i:arts i:sciences) -i:s","i:as -i:s"); + var tester = new RuleBaseTester("stemming.sr"); + tester.assertSemantics("+(AND i:arts i:sciences TRUE) -i:s", "i:as -i:s"); + } + + @Test + public void testEnglishStemming() { + var tester = new RuleBaseTester("stemming.sr"); + tester.assertSemantics("i:drive", "i:going"); } @Test - public void testNoRewritingBecauseShortWordsAreNotStemmed() { - assertSemantics("+i:a -i:s","i:a -i:s"); + public void testFrenchStemming() { + var tester = new RuleBaseTester("stemming-french.sr"); + tester.assertSemantics("i:going", "i:going"); } } diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/music.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/music.sr new file mode 100644 index 00000000000..5166a8b8e5d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/music.sr @@ -0,0 +1,19 @@ +## Some test rules + +# Spelling correction +bahc -> bach; + +# Stopwords +somelongstopword -> ; +[stopword] -> ; +[stopword] :- someotherlongstopword, yetanotherstopword; + +# +[song] by [artist] -> song:[song] artist:[artist]; + +[song] :- together, imagine, tinseltown; +[artist] :- youngbloods, beatles, zappa; + +# Negative +various +> -kingz; + diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-french.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-french.sr new file mode 100644 index 00000000000..1ccafd04344 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-french.sr @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@stemming(true) +@language(fr) + +i:as -> i:arts i:sciences; +i:car -> i:vehicle; +i:horses -> i:animal; +i:go -> i:drive; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-none.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-none.sr new file mode 100644 index 00000000000..44f6e40a308 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming-none.sr @@ -0,0 +1,6 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@stemming(false) + +i:car -> i:vehicle; +i:horses -> i:animal; +i:go -> i:drive; diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr index f68706646c2..ea73e385b3a 100644 --- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr +++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/rulebases/stemming.sr @@ -1,5 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @stemming(true) + i:as -> i:arts i:sciences; i:car -> i:vehicle; i:horses -> i:animal; +i:go -> i:drive; diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java index 2fda56c7454..4a7c6179db7 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/GroupingExecutorTestCase.java @@ -9,6 +9,7 @@ import com.yahoo.document.GlobalId; import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.NullItem; import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -535,18 +536,14 @@ public class GroupingExecutorTestCase { Execution exc = newExecution(new GroupingExecutor()); Query query = new Query(); - NotItem notItem = new NotItem(); - - notItem.addNegativeItem(new WordItem("negative")); - query.getModel().getQueryTree().setRoot(notItem); + query.getModel().getQueryTree().setRoot(new NullItem()); Result result = exc.search(query); com.yahoo.search.result.ErrorMessage message = result.hits().getError(); assertNotNull("Got error", message); assertEquals("Illegal query", message.getMessage()); - assertEquals("Can not search for only negative items", - message.getDetailedMessage()); + assertEquals("No query", message.getDetailedMessage()); assertEquals(3, message.getCode()); } diff --git a/container-search/src/test/java/com/yahoo/search/query/textserialize/item/test/ParseItemTestCase.java b/container-search/src/test/java/com/yahoo/search/query/textserialize/item/test/ParseItemTestCase.java index aa0d692ec92..b351bfb927a 100644 --- a/container-search/src/test/java/com/yahoo/search/query/textserialize/item/test/ParseItemTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/textserialize/item/test/ParseItemTestCase.java @@ -13,6 +13,7 @@ import com.yahoo.prelude.query.PrefixItem; import com.yahoo.prelude.query.RankItem; import com.yahoo.prelude.query.SubstringItem; import com.yahoo.prelude.query.SuffixItem; +import com.yahoo.prelude.query.TrueItem; import com.yahoo.prelude.query.WordItem; import com.yahoo.search.query.textserialize.item.ItemContext; import com.yahoo.search.query.textserialize.item.ItemFormHandler; @@ -71,7 +72,7 @@ public class ParseItemTestCase { @Test public void parse_and_not_rest_with_only_negated_children() throws ParseException { NotItem notItem = (NotItem) parse("(AND-NOT-REST null (WORD 'negated-item'))"); - assertNull(notItem.getPositiveItem()); + assertTrue(notItem.getPositiveItem() instanceof TrueItem); assertTrue(notItem.getItem(1) instanceof WordItem); } diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java index ef36e16a2b7..14fdd047391 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java @@ -164,7 +164,7 @@ public class ValidateNearestNeighborTestCase { String q = makeQuery("dvector", "foo"); Tensor t = makeTensor(tt_dense_dvector_3); Result r = doSearch(searcher, q, t); - assertErrMsg(desc("dvector", "foo", 1, "requires a tensor rank feature query(foo) but this is not present"), r); + assertErrMsg(desc("dvector", "foo", 1, "requires a tensor rank feature named 'query(foo)' but this is not present"), r); } @Test diff --git a/container-search/src/test/java/com/yahoo/search/yql/ParameterListParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/ParameterListParserTestCase.java new file mode 100644 index 00000000000..44f784e96f3 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/ParameterListParserTestCase.java @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import com.yahoo.prelude.query.WeightedSetItem; +import org.junit.Test; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class ParameterListParserTestCase { + + @Test + public void testMapParsing() { + assertParsed("{}", Map.of()); + assertParsed("{a:12}", Map.of("a", 12)); + assertParsed("{'a':12}", Map.of("a", 12)); + assertParsed("{\"a\":12}", Map.of("a", 12)); + assertParsed("{a:12,b:13}", Map.of("a", 12, "b", 13)); + assertParsed("{a:12, b:13}", Map.of("a", 12, "b", 13)); + assertParsed(" { a:12, b:13} ", Map.of("a", 12, "b", 13)); + assertParsed("{a:12, 'b':13} ", Map.of("a", 12, "b", 13)); + assertParsed("{a:12,'b':13, \"c,}\": 14}", Map.of("a", 12, "b", 13, "c,}", 14)); + } + + @Test + public void testArrayParsing() { + assertParsed("[]", Map.of()); + assertParsed("[[0,12]]", Map.of(0L, 12)); + assertParsed("[[0,12],[1,13]]", Map.of(0L, 12, 1L, 13)); + assertParsed("[[0,12], [1,13]]", Map.of(0L, 12, 1L, 13)); + assertParsed(" [ [0,12], [ 1,13]] ", Map.of(0L, 12, 1L, 13)); + } + + private void assertParsed(String string, Map<?, Integer> expected) { + WeightedSetItem item = new WeightedSetItem("test"); + ParameterListParser.addItemsFromString(string, item); + for (var entry : expected.entrySet()) { + assertEquals("Key '" + entry.getKey() + "'", entry.getValue(), item.getTokenWeight(entry.getKey())); + } + assertEquals("Token count is correct", expected.size(), item.getNumTokens()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java new file mode 100644 index 00000000000..efaaaa5fca7 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java @@ -0,0 +1,78 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.yql; + +import com.yahoo.component.chain.Chain; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.searchchain.Execution; +import org.apache.http.client.utils.URIBuilder; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Tests YQL expressions where a list of terms are supplied by indirection + * + * @author bratseth + */ +public class TermListTestCase { + + @Test + public void testTermListInWeightedSet() { + URIBuilder builder = searchUri(); + builder.setParameter("myTerms", "{'1':1, '2':1, 3:1}"); + builder.setParameter("yql", "select * from sources * where weightedSet(user_id, @myTerms)"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where weightedSet(user_id, {\"1\": 1, \"2\": 1, \"3\": 1});", + query.yqlRepresentation()); + } + + @Test + public void testTermListInWand() { + URIBuilder builder = searchUri(); + builder.setParameter("myTerms", "{'1':1, 2:1, '3':1}"); + builder.setParameter("yql", "select * from sources * where wand(user_id, @myTerms)"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where wand(user_id, {\"1\": 1, \"2\": 1, \"3\": 1});", + query.yqlRepresentation()); + } + + @Test + public void testTermListInDotProduct() { + URIBuilder builder = searchUri(); + builder.setParameter("myTerms", "{'1':1, '2':1, '3':1}"); + builder.setParameter("yql", "select * from sources * where dotProduct(user_id, @myTerms)"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where dotProduct(user_id, {\"1\": 1, \"2\": 1, \"3\": 1});", + query.yqlRepresentation()); + } + + private Query searchAndAssertNoErrors(URIBuilder builder) { + Query query = new Query(builder.toString()); + var searchChain = new Chain<>(new MinimalQueryInserter()); + var context = Execution.Context.createContextStub(); + var execution = new Execution(searchChain, context); + Result r = execution.search(query); + var exception = exceptionOf(r); + assertNull(exception == null ? "No error": + exception.getMessage() + "\n" + Arrays.toString(exception.getStackTrace()), + r.hits().getError()); + return query; + } + + private Throwable exceptionOf(Result r) { + if (r.hits().getError() == null) return null; + if (r.hits().getError().getCause() == null) return null; + return r.hits().getError().getCause(); + } + + private URIBuilder searchUri() { + URIBuilder builder = new URIBuilder(); + builder.setPath("search/"); + return builder; + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java index cc1a275af5a..c41de3a73f1 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java @@ -72,7 +72,7 @@ public class UserInputTestCase { @Test public void testRawUserInput() { URIBuilder builder = searchUri(); - builder.setParameter("yql", "select * from sources * where [{grammar: \"raw\"}]userInput(\"nal le\")"); + builder.setParameter("yql", "select * from sources * where {grammar: \"raw\"}userInput(\"nal le\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains \"nal le\";", query.yqlRepresentation()); } @@ -81,7 +81,7 @@ public class UserInputTestCase { public void testSegmentedUserInput() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{grammar: \"segment\"}]userInput(\"nal le\")"); + "select * from sources * where {grammar: \"segment\"}userInput(\"nal le\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains ([{origin: {original: \"nal le\", offset: 0, length: 6}}]phrase(\"nal\", \"le\"));", query.yqlRepresentation()); } @@ -90,12 +90,50 @@ public class UserInputTestCase { public void testSegmentedNoiseUserInput() { URIBuilder builder = searchUri(); builder.setParameter("yql", - "select * from sources * where [{grammar: \"segment\"}]userInput(\"^^^^^^^^\")"); + "select * from sources * where {grammar: \"segment\"}userInput(\"^^^^^^^^\")"); Query query = searchAndAssertNoErrors(builder); assertEquals("select * from sources * where default contains \"^^^^^^^^\";", query.yqlRepresentation()); } @Test + public void testAnyParsedUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", "select * from sources * where {grammar: \"any\"}userInput('foo bar')"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where (default contains \"foo\" OR default contains \"bar\");", + query.yqlRepresentation()); + } + + @Test + public void testAllParsedUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", "select * from sources * where {grammar: \"all\"}userInput('foo bar')"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where (default contains \"foo\" AND default contains \"bar\");", + query.yqlRepresentation()); + } + + @Test + public void testWeakAndParsedUserInput() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", "select * from sources * where {grammar: \"weakAnd\"}userInput('foo bar')"); + Query query = searchAndAssertNoErrors(builder); + assertEquals("select * from sources * where weakAnd(default contains \"foo\", default contains \"bar\");", + query.yqlRepresentation()); + } + + @Test + public void testIllegalGrammar() { + URIBuilder builder = searchUri(); + builder.setParameter("yql", "select * from sources * where {grammar: \"nonesuch\"}userInput('foo bar')"); + Query query = new Query(builder.toString()); + Result r = execution.search(query); + assertNotNull(r.hits().getError()); + assertEquals("Could not create query from YQL: No query type 'nonesuch'", + r.hits().getError().getDetailedMessage()); + } + + @Test public void testCustomDefaultIndexUserInput() { URIBuilder builder = searchUri(); builder.setParameter("yql", diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java index 27959948536..5d3a95efc78 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlFieldAndSourceTestCase.java @@ -27,7 +27,7 @@ import com.yahoo.search.searchchain.testutil.DocumentSourceSearcher; import static com.yahoo.search.searchchain.testutil.DocumentSourceSearcher.DEFAULT_SUMMARY_CLASS;; /** - * Test translation of fields and sources in YQL+ to the associated concepts in Vespa. + * Test translation of fields and sources in YQL to the associated concepts in Vespa. */ public class YqlFieldAndSourceTestCase { @@ -40,7 +40,6 @@ public class YqlFieldAndSourceTestCase { private Execution.Context context; private Execution execution; - @Before public void setUp() throws Exception { Query query = new Query("?query=test"); @@ -137,6 +136,7 @@ public class YqlFieldAndSourceTestCase { assertFalse(result.hits().get(0).isFilled(DEFAULT_SUMMARY_CLASS)); assertFalse(result.hits().get(0).isFilled(Execution.ATTRIBUTEPREFETCH)); } + @Test public final void testTrivialCaseWithOnlyDiskfieldWrongClassRequested() { final Query query = new Query("?query=test&presentation.summaryFields=" + FIELD1); diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 55fb53b4460..15713dc1f97 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -193,6 +193,18 @@ public class YqlParserTestCase { } @Test + public void testSingleNot() { + assertParse("select foo from bar where !(title contains \"saint\")", + "-title:saint"); + } + + @Test + public void testMultipleNot() { + assertParse("select foo from bar where !(title contains \"saint\") AND !(title contains \"etienne\")", + "-title:saint -title:etienne"); + } + + @Test public void testLessThan() { assertParse("select foo from bar where price < 500", "price:<500"); assertParse("select foo from bar where 500 < price", "price:>500"); @@ -387,7 +399,7 @@ public class YqlParserTestCase { assertFalse(root instanceof WordItem); assertTrue(root instanceof PhraseSegmentItem); - root = parse("select foo from bar where baz contains ([{grammar:\"raw\"}]\"yoni jo dima\")").getRoot(); + root = parse("select foo from bar where baz contains ({grammar:\"raw\"}\"yoni jo dima\")").getRoot(); assertEquals("baz:yoni jo dima", root.toString()); assertTrue(root instanceof WordItem); assertFalse(root instanceof ExactStringItem); @@ -398,7 +410,7 @@ public class YqlParserTestCase { AndItem andItem = (AndItem) root; assertEquals(3, andItem.getItemCount()); - root = parse("select foo from bar where [{grammar:\"raw\"}]userInput(\"yoni jo dima\")").getRoot(); + root = parse("select foo from bar where {grammar:\"raw\"}userInput(\"yoni jo dima\")").getRoot(); assertTrue(root instanceof WordItem); assertTrue(root instanceof ExactStringItem); assertEquals("yoni jo dima", ((WordItem)root).getWord()); @@ -407,9 +419,9 @@ public class YqlParserTestCase { @Test public void testAccentDropping() { assertFalse(getRootWord("select foo from bar where baz contains " + - "([ {accentDrop: false} ]\"colors\")").isNormalizable()); + "( {accentDrop: false} \"colors\")").isNormalizable()); assertTrue(getRootWord("select foo from bar where baz contains " + - "([ {accentDrop: true} ]\"colors\")").isNormalizable()); + "( {accentDrop: true} \"colors\")").isNormalizable()); assertTrue(getRootWord("select foo from bar where baz contains " + "\"colors\"").isNormalizable()); } @@ -417,9 +429,9 @@ public class YqlParserTestCase { @Test public void testCaseNormalization() { assertTrue(getRootWord("select foo from bar where baz contains " + - "([ {normalizeCase: false} ]\"colors\")").isLowercased()); + "( {normalizeCase: false} \"colors\")").isLowercased()); assertFalse(getRootWord("select foo from bar where baz contains " + - "([ {normalizeCase: true} ]\"colors\")").isLowercased()); + "( {normalizeCase: true} \"colors\")").isLowercased()); assertFalse(getRootWord("select foo from bar where baz contains " + "\"colors\"").isLowercased()); } @@ -428,10 +440,10 @@ public class YqlParserTestCase { public void testSegmentingRule() { assertEquals(SegmentingRule.PHRASE, getRootWord("select foo from bar where baz contains " + - "([ {andSegmenting: false} ]\"colors\")").getSegmentingRule()); + "( {andSegmenting: false} \"colors\")").getSegmentingRule()); assertEquals(SegmentingRule.BOOLEAN_AND, getRootWord("select foo from bar where baz contains " + - "([ {andSegmenting: true} ]\"colors\")").getSegmentingRule()); + "( {andSegmenting: true} \"colors\")").getSegmentingRule()); assertEquals(SegmentingRule.LANGUAGE_DEFAULT, getRootWord("select foo from bar where baz contains " + "\"colors\"").getSegmentingRule()); @@ -441,10 +453,10 @@ public class YqlParserTestCase { public void testNfkc() { assertEquals("a\u030a", getRootWord("select foo from bar where baz contains " + - "([ {nfkc: false} ]\"a\\u030a\")").getWord()); + "( {nfkc: false} \"a\\u030a\")").getWord()); assertEquals("\u00e5", getRootWord("select foo from bar where baz contains " + - "([ {nfkc: true} ]\"a\\u030a\")").getWord()); + "( {nfkc: true} \"a\\u030a\")").getWord()); assertEquals("No NKFC by default", "a\u030a", getRootWord("select foo from bar where baz contains " + @@ -453,19 +465,19 @@ public class YqlParserTestCase { @Test public void testImplicitTransforms() { - assertFalse(getRootWord("select foo from bar where baz contains ([ {implicitTransforms: " + - "false} ]\"cox\")").isFromQuery()); - assertTrue(getRootWord("select foo from bar where baz contains ([ {implicitTransforms: " + - "true} ]\"cox\")").isFromQuery()); + assertFalse(getRootWord("select foo from bar where baz contains ({implicitTransforms: " + + "false} \"cox\")").isFromQuery()); + assertTrue(getRootWord("select foo from bar where baz contains ({implicitTransforms: " + + "true} \"cox\")").isFromQuery()); assertTrue(getRootWord("select foo from bar where baz contains \"cox\"").isFromQuery()); } @Test public void testConnectivity() { QueryTree parsed = parse("select foo from bar where " + - "title contains ([{id: 1, connectivity: {\"id\": 3, weight: 7.0}}]\"madonna\") " + - "and title contains ([{id: 2}]\"saint\") " + - "and title contains ([{id: 3}]\"angel\")"); + "title contains ({id: 1, connectivity: {\"id\": 3, weight: 7.0}}\"madonna\") " + + "and title contains ({id: 2}\"saint\") " + + "and title contains ({id: 3}\"angel\")"); assertEquals("AND title:madonna title:saint title:angel", parsed.toString()); AndItem root = (AndItem)parsed.getRoot(); @@ -477,7 +489,7 @@ public class YqlParserTestCase { assertNull(second.getConnectedItem()); assertParseFail("select foo from bar where " + - "title contains ([{id: 1, connectivity: {id: 4, weight: 7.0}}]\"madonna\") " + + "title contains ({id: 1, connectivity: {id: 4, weight: 7.0}}\"madonna\") " + "and title contains ({id: 2}\"saint\") " + "and title contains ({id: 3}\"angel\")", new IllegalArgumentException("Item 'title:madonna' was specified to connect to item with ID 4, " + @@ -1241,4 +1253,5 @@ public class YqlParserTestCase { actual.add(step.continuations().toString() + step.getOperation()); return actual.toString(); } + } diff --git a/container-test/pom.xml b/container-test/pom.xml index 377272d939c..7c739faad26 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -26,6 +26,10 @@ <groupId>org.apache.httpcomponents.client5</groupId> <artifactId>httpclient5</artifactId> </exclusion> + <exclusion> + <groupId>biz.aQute.bnd</groupId> + <artifactId>*</artifactId> + </exclusion> </exclusions> </dependency> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index d4e11163343..b36d7880506 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -3,9 +3,9 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AccessControlService; -import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.CloudEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; +import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; @@ -28,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMoni import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; -import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretService; import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; @@ -50,8 +49,6 @@ public interface ServiceRegistry { NameService nameService(); - GlobalRoutingService globalRoutingService(); - Mailer mailer(); EndpointCertificateProvider endpointCertificateProvider(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java index a0dee6c059f..3a863f21ca0 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java @@ -175,7 +175,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { return Boolean.compare(buildNumber().isPresent(), o.buildNumber.isPresent()); // Unknown version sorts first if (deployedDirectly || o.deployedDirectly) - return Boolean.compare(deployedDirectly, o.deployedDirectly); // Directly deployed versions sort first + return Boolean.compare( ! deployedDirectly, ! o.deployedDirectly); // Directly deployed versions sort first return Long.compare(buildNumber().getAsLong(), o.buildNumber().getAsLong()); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java index 4f2ab2b1734..c06ade1adcf 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java @@ -15,7 +15,7 @@ public interface ArtifactRepository { /** Returns the system application package of the given version. */ byte[] getSystemApplicationPackage(ApplicationId application, ZoneId zone, Version version); - /** Returns the current stable OS version for the given major version */ - StableOsVersion stableOsVersion(int major); + /** Returns the current OS release with the given major version and tag */ + OsRelease osRelease(int major, OsRelease.Tag tag); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/OsRelease.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/OsRelease.java new file mode 100644 index 00000000000..d80b2201810 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/OsRelease.java @@ -0,0 +1,65 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.deployment; + +import com.yahoo.component.Version; + +import java.time.Instant; +import java.util.Objects; + +/** + * An OS release and its tag. + * + * @author mpolden + */ +public class OsRelease { + + private final Version version; + private final Tag tag; + private final Instant taggedAt; + + public OsRelease(Version version, Tag tag, Instant taggedAt) { + this.version = Objects.requireNonNull(version); + this.tag = Objects.requireNonNull(tag); + this.taggedAt = Objects.requireNonNull(taggedAt); + } + + /** The version number */ + public Version version() { + return version; + } + + /** The tag of this */ + public Tag tag() { + return tag; + } + + /** Returns the time this was tagged */ + public Instant taggedAt() { + return taggedAt; + } + + @Override + public String toString() { + return "os release " + version + ", tagged " + tag + " at " + taggedAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OsRelease osRelease = (OsRelease) o; + return version.equals(osRelease.version) && tag == osRelease.tag && taggedAt.equals(osRelease.taggedAt); + } + + @Override + public int hashCode() { + return Objects.hash(version, tag, taggedAt); + } + + /** Known release tags */ + public enum Tag { + latest, + stable, + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/StableOsVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/StableOsVersion.java deleted file mode 100644 index 98bf5d9d0d7..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/StableOsVersion.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.deployment; - -import com.yahoo.component.Version; - -import java.time.Instant; -import java.util.Objects; - -/** - * A stable OS version. - * - * @author mpolden - */ -public class StableOsVersion { - - private final Version version; - private final Instant promotedAt; - - public StableOsVersion(Version version, Instant promotedAt) { - this.version = Objects.requireNonNull(version); - this.promotedAt = Objects.requireNonNull(promotedAt); - } - - /** The version number */ - public Version version() { - return version; - } - - /** Returns the time this was promoted to stable */ - public Instant promotedAt() { - return promotedAt; - } - - @Override - public String toString() { - return "os version " + version + ", promoted at " + promotedAt; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - StableOsVersion that = (StableOsVersion) o; - return version.equals(that.version) && promotedAt.equals(that.promotedAt); - } - - @Override - public int hashCode() { - return Objects.hash(version, promotedAt); - } - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java deleted file mode 100644 index e14a5a5d562..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/GlobalRoutingService.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.routing; - -import com.yahoo.config.provision.zone.ZoneId; - -import java.util.Map; - -/** - * A service containing the health status of global rotations. - * - * @author mpolden - */ -public interface GlobalRoutingService { - - /** Returns the health status of each zone behind the given rotation name */ - Map<ZoneId, RotationStatus> getHealthStatus(String rotationName); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/MemoryGlobalRoutingService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/MemoryGlobalRoutingService.java deleted file mode 100644 index 5d51030a329..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/MemoryGlobalRoutingService.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.routing; - -import com.yahoo.component.AbstractComponent; -import com.yahoo.config.provision.zone.ZoneId; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -/** - * @author bratseth - */ -public class MemoryGlobalRoutingService extends AbstractComponent implements GlobalRoutingService { - - private final Map<String, Map<ZoneId, RotationStatus>> status = new HashMap<>(); - - @Override - public Map<ZoneId, RotationStatus> getHealthStatus(String rotationName) { - if (status.isEmpty()) { - return Map.of(ZoneId.from("prod", "us-west-1"), RotationStatus.IN); - } - return Collections.unmodifiableMap(status.getOrDefault(rotationName, Map.of())); - } - - public MemoryGlobalRoutingService setStatus(String rotation, ZoneId zone, RotationStatus status) { - this.status.putIfAbsent(rotation, new HashMap<>()); - this.status.get(rotation).put(zone, status); - return this; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index a23cb40dcb1..d710a8a2948 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -370,7 +370,6 @@ public class ApplicationController { try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); Instance instance = application.get().require(job.application().instance()); - rejectOldChange(instance, platform, revision, job, zone); if ( ! applicationPackage.trustedCertificates().isEmpty() && run.testerCertificate().isPresent()) @@ -801,20 +800,6 @@ public class ApplicationController { } } - private void rejectOldChange(Instance instance, Version platform, ApplicationVersion revision, JobId job, ZoneId zone) { - Deployment deployment = instance.deployments().get(zone); - if (deployment == null) return; - if (!zone.environment().isProduction()) return; - - boolean platformIsOlder = platform.compareTo(deployment.version()) < 0 && !instance.change().isPinned(); - boolean revisionIsOlder = revision.compareTo(deployment.applicationVersion()) < 0 && - !(revision.isUnknown() && controller.system().isCd()); - if (platformIsOlder || revisionIsOlder) - throw new IllegalArgumentException(Text.format("Rejecting deployment of application %s to %s, as the requested versions (platform: %s, application: %s)" + - " are older than the currently deployed (platform: %s, application: %s).", - job.application(), zone, platform, revision, deployment.version(), deployment.applicationVersion())); - } - private TenantAndApplicationId dashToUnderscore(TenantAndApplicationId id) { return TenantAndApplicationId.from(id.tenant().value(), id.application().value().replaceAll("-", "_")); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 943d6ac7b18..daf9cfd428f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -441,16 +441,10 @@ public class RoutingController { .on(Port.fromRoutingMethod(method)) .routingMethod(method) .in(controller.system())); - // Add legacy endpoints + // Add legacy endpoint if (legacyNamesAvailable && method == RoutingMethod.shared) { endpoints.add(Endpoint.of(routingId.instance()) .target(routingId.endpointId(), cluster, deployments) - .on(Port.plain(4080)) - .legacy() - .routingMethod(method) - .in(controller.system())); - endpoints.add(Endpoint.of(routingId.instance()) - .target(routingId.endpointId(), cluster, deployments) .on(Port.tls(4443)) .legacy() .routingMethod(method) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index a6e53fead37..5c0669ad543 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -61,6 +61,12 @@ public class ApplicationList extends AbstractFilteringList<Application, Applicat .anyMatch(deployment -> deployment.version().isBefore(version))); } + /** Returns the subset of applications with at least one declared job in deployment spec. */ + public ApplicationList withJobs() { + return matching(application -> application.deploymentSpec().steps().stream() + .anyMatch(step -> ! step.zones().isEmpty())); + } + /** Returns the subset of applications which have a project ID */ public ApplicationList withProjectId() { return matching(application -> application.projectId().isPresent()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 8f37d287c1a..262730558d0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -42,7 +42,6 @@ public class Endpoint { private final Scope scope; private final boolean legacy; private final RoutingMethod routingMethod; - private final boolean tls; private Endpoint(TenantAndApplicationId application, Optional<InstanceName> instanceName, EndpointId id, ClusterSpec.Id cluster, URI url, List<Target> targets, Scope scope, Port port, boolean legacy, @@ -63,7 +62,6 @@ public class Endpoint { this.scope = requireScope(scope, routingMethod); this.legacy = legacy; this.routingMethod = routingMethod; - this.tls = port.tls; } /** @@ -125,7 +123,7 @@ public class Endpoint { /** Returns whether this endpoint supports TLS connections */ public boolean tls() { - return tls; + return true; } /** Returns whether this requires a rotation to be reachable */ @@ -164,10 +162,9 @@ public class Endpoint { private static URI createUrl(String name, TenantAndApplicationId application, Optional<InstanceName> instance, List<Target> targets, Scope scope, SystemName system, Port port, boolean legacy, RoutingMethod routingMethod) { - String scheme = port.tls ? "https" : "http"; - String separator = separator(system, routingMethod, port.tls); + String separator = separator(system, routingMethod); String portPart = port.isDefault() ? "" : ":" + port.port; - return URI.create(scheme + "://" + + return URI.create("https://" + sanitize(namePart(name, separator)) + systemPart(system, separator) + sanitize(instancePart(instance, separator)) + @@ -185,8 +182,7 @@ public class Endpoint { return part.replace('_', '-'); } - private static String separator(SystemName system, RoutingMethod routingMethod, boolean tls) { - if (!tls) return "."; + private static String separator(SystemName system, RoutingMethod routingMethod) { if (routingMethod.isDirect()) return "."; if (system.isPublic()) return "."; return "--"; @@ -390,21 +386,19 @@ public class Endpoint { /** Represents an endpoint's HTTP port */ public static class Port { - private static final Port TLS_DEFAULT = new Port(443, true); + private static final Port TLS_DEFAULT = new Port(443); private final int port; - private final boolean tls; - private Port(int port, boolean tls) { + private Port(int port) { if (port < 1 || port > 65535) { throw new IllegalArgumentException("Port must be between 1 and 65535, got " + port); } this.port = port; - this.tls = tls; } private boolean isDefault() { - return port == 80 || port == 443; + return port == 443; } /** Returns the default HTTPS port */ @@ -420,12 +414,7 @@ public class Endpoint { /** Create a HTTPS port */ public static Port tls(int port) { - return new Port(port, true); - } - - /** Create a HTTP port */ - public static Port plain(int port) { - return new Port(port, false); + return new Port(port); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index ce03e84f2b9..e7521b37dbf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -612,6 +613,9 @@ public class DeploymentStatus { && ! existingDeployment.map(Deployment::version).equals(change.platform())) return Optional.empty(); + if (change.application().isPresent() && ! existingDeployment.map(Deployment::applicationVersion).equals(change.application())) + return Optional.empty(); + Change fullChange = status.application().require(instance).change(); if (existingDeployment.map(deployment -> ! (change.upgrades(deployment.version()) || change.upgrades(deployment.applicationVersion())) && (fullChange.downgrades(deployment.version()) || fullChange.downgrades(deployment.applicationVersion()))) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 0d56bc286eb..684c497571d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -463,14 +463,13 @@ public class InternalStepRunner implements StepRunner { if ( ! endpoints.containsKey(zoneId)) return false; - return endpoints.get(zoneId).parallelStream().map(endpoint -> { + return endpoints.get(zoneId).parallelStream().allMatch(endpoint -> { boolean ready = controller.jobController().cloud().ready(endpoint.url()); - if ( ! ready) { + if (!ready) { logger.log("Failed to get 100 consecutive OKs from " + endpoint); - return Boolean.FALSE; } - return Boolean.TRUE; - }).allMatch(Boolean.TRUE::equals); + return ready; + }); } /** Returns true iff all containers in the tester deployment give 100 consecutive 200 OK responses on /status.html. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 2480c81755e..1601af2612b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -396,22 +396,23 @@ public class JobController { }); logs.flush(id); metric.jobFinished(run.id().job(), finishedRun.status()); + + DeploymentId deploymentId = new DeploymentId(unlockedRun.id().application(), unlockedRun.id().job().type().zone(controller.system())); + (unlockedRun.versions().targetApplication().isDeployedDirectly() ? + Stream.of(unlockedRun.id().type()) : + JobType.allIn(controller.system()).stream().filter(jobType -> !jobType.environment().isManuallyDeployed())) + .flatMap(jobType -> controller.jobController().runs(unlockedRun.id().application(), jobType).values().stream()) + .mapToLong(r -> r.versions().targetApplication().buildNumber().orElse(Integer.MAX_VALUE)) + .min() + .ifPresent(oldestBuild -> { + if (unlockedRun.versions().targetApplication().isDeployedDirectly()) + controller.applications().applicationStore().pruneDevDiffs(deploymentId, oldestBuild); + else + controller.applications().applicationStore().pruneDiffs(deploymentId.applicationId().tenant(), deploymentId.applicationId().application(), oldestBuild); + }); + return finishedRun; }); - - DeploymentId deploymentId = new DeploymentId(unlockedRun.id().application(), unlockedRun.id().job().type().zone(controller.system())); - (unlockedRun.versions().targetApplication().isDeployedDirectly() ? - Stream.of(unlockedRun.id().type()) : - JobType.allIn(controller.system()).stream().filter(jobType -> !jobType.environment().isManuallyDeployed())) - .flatMap(jobType -> controller.jobController().runs(unlockedRun.id().application(), jobType).values().stream()) - .mapToLong(run -> run.versions().targetApplication().buildNumber().orElse(Integer.MAX_VALUE)) - .min() - .ifPresent(oldestBuild -> { - if (unlockedRun.versions().targetApplication().isDeployedDirectly()) - controller.applications().applicationStore().pruneDevDiffs(deploymentId, oldestBuild); - else - controller.applications().applicationStore().pruneDiffs(deploymentId.applicationId().tenant(), deploymentId.applicationId().application(), oldestBuild); - }); } finally { for (Lock lock : locks) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java index 779ce6fa7fe..1e183d44377 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java @@ -126,8 +126,9 @@ public class Versions { private static ApplicationVersion targetApplication(Application application, Change change, Optional<Deployment> deployment) { - return max(change.application(), deployment.map(Deployment::applicationVersion)) - .orElseGet(() -> defaultApplicationVersion(application)); + return change.application() + .or(() -> deployment.map(Deployment::applicationVersion)) + .orElseGet(() -> defaultApplicationVersion(application)); } private static ApplicationVersion defaultApplicationVersion(Application application) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index b2d3e1b780f..aa087f58059 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.StableOsVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.OsRelease; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import java.time.Duration; @@ -67,10 +67,10 @@ public class OsUpgradeScheduler extends ControllerMaintainer { } private Release releaseIn(CloudName cloud) { - boolean useStableRelease = controller().zoneRegistry().zones().reprovisionToUpgradeOs().ofCloud(cloud) + boolean useTaggedRelease = controller().zoneRegistry().zones().reprovisionToUpgradeOs().ofCloud(cloud) .zones().isEmpty(); - if (useStableRelease) { - return new StableRelease(controller().system(), controller().serviceRegistry().artifactRepository()); + if (useTaggedRelease) { + return new TaggedRelease(controller().system(), controller().serviceRegistry().artifactRepository()); } return new CalendarVersionedRelease(controller().system()); } @@ -85,32 +85,37 @@ public class OsUpgradeScheduler extends ControllerMaintainer { } - /** OS release based on a stable tag */ - private static class StableRelease implements Release { + /** OS release based on a tag */ + private static class TaggedRelease implements Release { private final SystemName system; private final ArtifactRepository artifactRepository; - private StableRelease(SystemName system, ArtifactRepository artifactRepository) { + private TaggedRelease(SystemName system, ArtifactRepository artifactRepository) { this.system = Objects.requireNonNull(system); this.artifactRepository = Objects.requireNonNull(artifactRepository); } @Override public Version version(OsVersionTarget currentTarget, Instant now) { - StableOsVersion stableVersion = artifactRepository.stableOsVersion(currentTarget.osVersion().version().getMajor()); - boolean cooldownPassed = stableVersion.promotedAt().isBefore(now.minus(cooldown())); - return cooldownPassed ? stableVersion.version() : currentTarget.osVersion().version(); + OsRelease release = artifactRepository.osRelease(currentTarget.osVersion().version().getMajor(), tag()); + boolean cooldownPassed = !release.taggedAt().plus(cooldown()).isAfter(now); + return cooldownPassed ? release.version() : currentTarget.osVersion().version(); } @Override public Duration upgradeBudget() { - return Duration.ZERO; // Stable releases happen in-place so no budget is required + return Duration.ZERO; // Upgrades to tagged releases happen in-place so no budget is required } - /** The cool-down period that must pass before a stable version can be used */ + /** Returns the release tag tracked by this system */ + private OsRelease.Tag tag() { + return system.isCd() ? OsRelease.Tag.latest : OsRelease.Tag.stable; + } + + /** The cool-down period that must pass before a release can be used */ private Duration cooldown() { - return system.isCd() ? Duration.ZERO : Duration.ofDays(7); + return system.isCd() ? Duration.ofDays(1) : Duration.ZERO; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 99ab6d420cb..8d5851be62f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -34,7 +34,7 @@ public class SystemUpgrader extends InfrastructureUpgrader<VespaVersionTarget> { @Override protected void upgrade(VespaVersionTarget target, SystemApplication application, ZoneApi zone) { - log.info(Text.format("Deploying %s version %s in %s", application.id(), target, zone.getId())); + log.info(Text.format("Deploying %s on %s in %s", application.id(), target, zone.getId())); controller().applications().deploy(application, zone.getId(), target.version(), target.downgrade()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index f18229b4377..b5008a44c6d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -94,7 +94,8 @@ public class DeploymentApiHandler extends LoggingRequestHandler { Cursor platformArray = root.setArray("versions"); var versionStatus = controller.readVersionStatus(); var systemVersion = controller.systemVersion(versionStatus); - var deploymentStatuses = controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()), systemVersion); + ApplicationList applications = ApplicationList.from(controller.applications().asList()).withJobs(); + var deploymentStatuses = controller.jobController().deploymentStatuses(applications, systemVersion); var deploymentStatistics = DeploymentStatistics.compute(versionStatus.versions().stream().map(VespaVersion::versionNumber).collect(toList()), deploymentStatuses) .stream().collect(toMap(DeploymentStatistics::version, identity())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index be8e49cf661..9f6f7e4dd5c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -96,13 +96,9 @@ public class RoutingPolicy { DeploymentId deployment = new DeploymentId(id.owner(), id.zone()); List<Endpoint> endpoints = new ArrayList<>(); endpoints.add(endpoint(routingMethod).target(id.cluster(), deployment).in(system)); - // Add legacy endpoints + // Add legacy endpoint if (routingMethod == RoutingMethod.shared) { endpoints.add(endpoint(routingMethod).target(id.cluster(), deployment) - .on(Port.plain(4080)) - .legacy() - .in(system)); - endpoints.add(endpoint(routingMethod).target(id.cluster(), deployment) .on(Port.tls(4443)) .legacy() .in(system)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 24a739d4fc4..238ab0b09fa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -121,7 +121,8 @@ public class VersionStatus { List<DeploymentStatistics> deploymentStatistics = DeploymentStatistics.compute(infrastructureVersions.keySet(), controller.jobController().deploymentStatuses(ApplicationList.from(controller.applications().asList()) - .withProjectId())); + .withProjectId() + .withJobs())); List<VespaVersion> versions = new ArrayList<>(); List<Version> releasedVersions = controller.mavenRepository().metadata().versions(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersionTarget.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersionTarget.java index fd5603b96b8..78d6d0ebf29 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersionTarget.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersionTarget.java @@ -30,4 +30,9 @@ public class VespaVersionTarget implements VersionTarget { return downgrade; } + @Override + public String toString() { + return "vespa version target " + version.toFullString() + (downgrade ? " (downgrade)" : ""); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 1215ddbc2ad..ce8c981fb5b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -334,12 +334,11 @@ public class ControllerTest { assertEquals("Rotation names are passed to config server in " + deployment.zone(), Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud", - "app1.tenant1.global.vespa.yahooapis.com", "app1--tenant1.global.vespa.yahooapis.com"), tester.configServer().containerEndpointNames(context.deploymentIdIn(deployment.zone()))); } context.flushDnsUpdates(); - assertEquals(3, tester.controllerTester().nameService().records().size()); + assertEquals(2, tester.controllerTester().nameService().records().size()); Optional<Record> record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); @@ -351,16 +350,10 @@ public class ControllerTest { assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); - List<String> globalDnsNames = tester.controller().routing().readDeclaredEndpointsOf(context.instanceId()) .scope(Endpoint.Scope.global) .mapToList(Endpoint::dnsName); assertEquals(List.of("app1--tenant1.global.vespa.oath.cloud", - "app1.tenant1.global.vespa.yahooapis.com", "app1--tenant1.global.vespa.yahooapis.com"), globalDnsNames); } @@ -1028,7 +1021,6 @@ public class ControllerTest { .scope(Endpoint.Scope.zone) .mapToList(Endpoint::dnsName); assertEquals(List.of("application--tenant.us-west-1.vespa.oath.cloud", - "application.tenant.us-west-1.prod.vespa.yahooapis.com", "application--tenant.us-west-1.prod.vespa.yahooapis.com", "application.tenant.us-west-1.vespa.oath.cloud"), zoneDnsNames); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index e50c32d0e5d..8e6618dc1a7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -33,10 +33,6 @@ public class EndpointTest { EndpointId endpointId = EndpointId.defaultId(); Map<String, Endpoint> tests = Map.of( - // Legacy endpoint - "http://a1.t1.global.vespa.yahooapis.com:4080/", - Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.plain(4080)).legacy().in(SystemName.main), - // Legacy endpoint with TLS "https://a1--t1.global.vespa.yahooapis.com:4443/", Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.tls(4443)).legacy().in(SystemName.main), @@ -99,10 +95,6 @@ public class EndpointTest { EndpointId endpointId = EndpointId.defaultId(); Map<String, Endpoint> tests = Map.of( - // Legacy endpoint - "http://a1.t1.global.vespa.yahooapis.com:4080/", - Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.plain(4080)).legacy().in(SystemName.main), - // Legacy endpoint with TLS "https://a1--t1.global.vespa.yahooapis.com:4443/", Endpoint.of(instance1).target(endpointId, cluster, List.of(deployment1)).on(Port.tls(4443)).legacy().in(SystemName.main), @@ -161,10 +153,6 @@ public class EndpointTest { var testZone = new DeploymentId(instance1, ZoneId.from("test", "us-north-2")); Map<String, Endpoint> tests = Map.of( - // Legacy endpoint (always contains environment) - "http://a1.t1.us-north-1.prod.vespa.yahooapis.com:4080/", - Endpoint.of(instance1).target(cluster, prodZone).on(Port.plain(4080)).legacy().in(SystemName.main), - // Secure legacy endpoint "https://a1--t1.us-north-1.prod.vespa.yahooapis.com:4443/", Endpoint.of(instance1).target(cluster, prodZone).on(Port.tls(4443)).legacy().in(SystemName.main), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 102dfde16ec..fd7ba8693e2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -465,6 +465,23 @@ public class DeploymentTriggerTest { } @Test + public void downgradingApplicationVersionWorks() { + var app = tester.newDeploymentContext().submit().deploy(); + ApplicationVersion appVersion0 = app.lastSubmission().get(); + app.submit().deploy(); + + // Downgrading application version. + tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion0)); + assertEquals(Change.of(appVersion0), app.instance().change()); + app.runJob(stagingTest) + .runJob(productionUsCentral1) + .runJob(productionUsEast3) + .runJob(productionUsWest1); + assertEquals(Change.empty(), app.instance().change()); + assertEquals(appVersion0, app.instance().deployments().get(productionUsEast3.zone(tester.controller().system())).applicationVersion()); + } + + @Test public void settingANoOpChangeIsANoOp() { var app = tester.newDeploymentContext().submit().deploy(); ApplicationVersion appVersion0 = app.lastSubmission().get(); @@ -473,8 +490,6 @@ public class DeploymentTriggerTest { // Triggering a roll-out of an already deployed application is a no-op. assertEquals(Change.empty(), app.instance().change()); - tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion0)); - assertEquals(Change.empty(), app.instance().change()); tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion1)); assertEquals(Change.empty(), app.instance().change()); } @@ -1114,9 +1129,10 @@ public class DeploymentTriggerTest { tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false); app.runJob(productionCdUsEast1) .abortJob(stagingTest) // Complete failing run. - .runJob(stagingTest) + .runJob(stagingTest) // Run staging-test for production zone with no prior deployment. .runJob(productionCdAwsUsEast1a); + // Manually deploy to east again, then upgrade the system. app.runJob(productionCdUsEast1, cdPackage); var version = new Version("7.1"); tester.controllerTester().upgradeSystem(version); @@ -1124,16 +1140,16 @@ public class DeploymentTriggerTest { // System and staging tests both require unknown versions, and are broken. tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false); app.runJob(productionCdUsEast1) - .jobAborted(systemTest) + .abortJob(systemTest) .jobAborted(stagingTest) - .runJob(systemTest) - .runJob(stagingTest) + .runJob(systemTest) // Run test for aws zone again. + .runJob(stagingTest) // Run test for aws zone again. .runJob(productionCdAwsUsEast1a); + // Deploy manually again, then submit new package. app.runJob(productionCdUsEast1, cdPackage); app.submit(cdPackage); - app.jobAborted(systemTest) - .runJob(systemTest); + app.runJob(systemTest); // Staging test requires unknown initial version, and is broken. tester.controller().applications().deploymentTrigger().forceTrigger(app.instanceId(), productionCdUsEast1, "user", false); app.runJob(productionCdUsEast1) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java index aa4cabb4fe8..dc7010312a2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java @@ -6,7 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.StableOsVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.OsRelease; import java.util.HashMap; import java.util.Map; @@ -16,7 +16,7 @@ import java.util.Map; */ public class ArtifactRepositoryMock extends AbstractComponent implements ArtifactRepository { - private final Map<Integer, StableOsVersion> stableOsVersions = new HashMap<>(); + private final Map<String, OsRelease> releases = new HashMap<>(); @Override public byte[] getSystemApplicationPackage(ApplicationId application, ZoneId zone, Version version) { @@ -24,14 +24,18 @@ public class ArtifactRepositoryMock extends AbstractComponent implements Artifac } @Override - public StableOsVersion stableOsVersion(int major) { - StableOsVersion version = stableOsVersions.get(major); - if (version == null) throw new IllegalArgumentException("No version set for major " + major); - return version; + public OsRelease osRelease(int major, OsRelease.Tag tag) { + OsRelease release = releases.get(key(major, tag)); + if (release == null) throw new IllegalArgumentException("No version set for major " + major + " with tag " + tag); + return release; } - public void promoteOsVersion(StableOsVersion stableOsVersion) { - stableOsVersions.put(stableOsVersion.version().getMajor(), stableOsVersion); + public void addRelease(OsRelease osRelease) { + releases.put(key(osRelease.version().getMajor(), osRelease.tag()), osRelease); + } + + private static String key(int major, OsRelease.Tag tag) { + return major + "@" + tag.name(); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index b81b3ae5d66..74c06d7ca1a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -11,15 +11,15 @@ import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.archive.MockArchiveService; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AccessControlService; import com.yahoo.vespa.hosted.controller.api.integration.athenz.MockAccessControlService; -import com.yahoo.vespa.hosted.controller.api.integration.aws.MockRoleService; -import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockCloudEventFetcher; import com.yahoo.vespa.hosted.controller.api.integration.aws.MockResourceTagger; +import com.yahoo.vespa.hosted.controller.api.integration.aws.MockRoleService; import com.yahoo.vespa.hosted.controller.api.integration.aws.ResourceTagger; +import com.yahoo.vespa.hosted.controller.api.integration.aws.RoleService; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClient; -import com.yahoo.vespa.hosted.controller.api.integration.billing.MockBillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClientMock; +import com.yahoo.vespa.hosted.controller.api.integration.billing.MockBillingController; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMock; @@ -32,10 +32,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.horizon.MockHorizonClie import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler; import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClientMock; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; -import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; -import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClientMock; import com.yahoo.vespa.hosted.controller.api.integration.secrets.NoopTenantSecretService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummySystemMonitor; @@ -59,7 +57,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final ZoneRegistryMock zoneRegistryMock; private final ConfigServerMock configServerMock; private final MemoryNameService memoryNameService = new MemoryNameService(); - private final MemoryGlobalRoutingService memoryGlobalRoutingService = new MemoryGlobalRoutingService(); private final MockMailer mockMailer = new MockMailer(); private final EndpointCertificateMock endpointCertificateMock = new EndpointCertificateMock(); private final EndpointCertificateValidatorMock endpointCertificateValidatorMock = new EndpointCertificateValidatorMock(); @@ -116,11 +113,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg } @Override - public GlobalRoutingService globalRoutingService() { - return memoryGlobalRoutingService; - } - - @Override public MockMailer mailer() { return mockMailer; } @@ -279,10 +271,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return configServerMock; } - public MemoryGlobalRoutingService globalRoutingServiceMock() { - return memoryGlobalRoutingService; - } - public MockContactRetriever contactRetrieverMock() { return mockContactRetriever; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 604a42f3d19..581ec68b3dd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -283,7 +283,7 @@ public class MetricsReporterTest { context.submit(applicationPackage).deploy(); reporter.maintain(); - assertEquals("Deployment queues name services requests", 6, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); + assertEquals("Deployment queues name services requests", 4, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); context.flushDnsUpdates(); reporter.maintain(); @@ -554,15 +554,6 @@ public class MetricsReporterTest { assertEquals("Upgrade is overdue measure relative to window 3", Duration.ofHours(34).plusMinutes(30), metric.get()); } - @Test - public void overdue_upgrade_completely_blocked() { - ApplicationPackage pkg = new ApplicationPackageBuilder().region("us-west-1") - .blockChange(false, true, "mon-sun", "0-23", "CET") - .build(); - Instant mondayNight = Instant.parse("2021-12-13T23:00:00.00Z"); - assertEquals(Duration.ZERO, MetricsReporter.overdueUpgradeDuration(mondayNight, pkg.deploymentSpec().requireInstance("default"))); - } - private void assertNodeCount(String metric, int n, Version version) { long nodeCount = metrics.getMetric((dimensions) -> version.toFullString().equals(dimensions.get("currentVersion")), metric) .stream() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java index 51bda73025d..300aa86b5ea 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java @@ -3,9 +3,10 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.StableOsVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.OsRelease; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import org.junit.Test; @@ -68,7 +69,6 @@ public class OsUpgradeSchedulerTest { @Test public void schedule_stable_release() { ControllerTester tester = new ControllerTester(); - OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)); Instant t0 = Instant.parse("2021-06-21T07:00:00.00Z"); // Inside trigger period tester.clock().setInstant(t0); @@ -77,40 +77,49 @@ public class OsUpgradeSchedulerTest { Version version0 = Version.fromString("8.0"); tester.controller().upgradeOsIn(cloud, version0, Duration.ZERO, false); - // New version is promoted to stable + // Stable release is scheduled immediately Version version1 = Version.fromString("8.1"); - tester.serviceRegistry().artifactRepository().promoteOsVersion(new StableOsVersion(version1, tester.clock().instant())); - scheduler.maintain(); - assertEquals("Target is unchanged as not enough time has passed", version0, - tester.controller().osVersionTarget(cloud).get().osVersion().version()); - - // Enough time passes since promotion of stable release - tester.clock().advance(Duration.ofDays(7).plus(Duration.ofSeconds(1))); - scheduler.maintain(); - OsVersionTarget target0 = tester.controller().osVersionTarget(cloud).get(); - assertEquals(version1, target0.osVersion().version()); - assertEquals("No budget when upgrading to stable release", - Duration.ZERO, target0.upgradeBudget()); - - // Another version is promoted, but target remains unchanged as the release hasn't aged enough - tester.clock().advance(Duration.ofDays(1)); - Version version2 = Version.fromString("8.2"); - tester.serviceRegistry().artifactRepository().promoteOsVersion(new StableOsVersion(version2, tester.clock().instant())); - scheduler.maintain(); - OsVersionTarget target1 = tester.controller().osVersionTarget(cloud).get(); - assertEquals("Target is unchanged as not enough time has passed", version1, - target1.osVersion().version()); - assertEquals("Target is not re-scheduled", target0.scheduledAt(), target1.scheduledAt()); + tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.stable, + tester.clock().instant())); + scheduleUpgradeAfter(Duration.ZERO, version1, tester); // A newer version is triggered manually Version version3 = Version.fromString("8.3"); tester.controller().upgradeOsIn(cloud, version3, Duration.ZERO, false); - // Enough time passes for stable version to be promoted. Nothing happens as stable is now before the manually - // triggered version - tester.clock().advance(Duration.ofDays(7).plus(Duration.ofSeconds(1))); - scheduler.maintain(); - assertEquals(version3, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + // Nothing happens in next iteration as tagged release is older than manually triggered version + scheduleUpgradeAfter(Duration.ofDays(7), version3, tester); + } + + @Test + public void schedule_latest_release_in_cd() { + ControllerTester tester = new ControllerTester(SystemName.cd); + Instant t0 = Instant.parse("2021-06-21T07:00:00.00Z"); // Inside trigger period + tester.clock().setInstant(t0); + + // Set initial target + CloudName cloud = tester.controller().clouds().iterator().next(); + Version version0 = Version.fromString("8.0"); + tester.controller().upgradeOsIn(cloud, version0, Duration.ZERO, false); + + // Latest release is not scheduled immediately + Version version1 = Version.fromString("8.1"); + tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.latest, + tester.clock().instant())); + scheduleUpgradeAfter(Duration.ZERO, version0, tester); + + // Cooldown period passes and latest release is scheduled + scheduleUpgradeAfter(Duration.ofDays(1), version1, tester); + } + + private void scheduleUpgradeAfter(Duration duration, Version version, ControllerTester tester) { + tester.clock().advance(duration); + new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)).maintain(); + CloudName cloud = tester.controller().clouds().iterator().next(); + OsVersionTarget target = tester.controller().osVersionTarget(cloud).get(); + assertEquals(version, target.osVersion().version()); + assertEquals("No budget when scheduling a tagged release", + Duration.ZERO, target.upgradeBudget()); } private static ZoneApi zone(String id, CloudName cloud) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 8eaa190e9fa..c78f83ced57 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -448,7 +448,6 @@ public class ApplicationApiTest extends ControllerContainerTest { deploymentTester.upgrader().overrideConfidence(Version.fromString("6.1"), VespaVersion.Confidence.broken); deploymentTester.controllerTester().computeVersionStatus(); setDeploymentMaintainedInfo(); - setZoneInRotation("rotation-fqdn-1", ZoneId.from("prod", "us-central-1")); // GET tenant application deployments tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) @@ -949,7 +948,6 @@ public class ApplicationApiTest extends ControllerContainerTest { 404); // GET global rotation status - setZoneInRotation("rotation-fqdn-1", ZoneId.from("prod", "us-west-1")); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET) .userIdentity(USER_ID), new File("global-rotation.json")); @@ -1001,10 +999,6 @@ public class ApplicationApiTest extends ControllerContainerTest { var app = deploymentTester.newDeploymentContext("tenant1", "application1", "instance1"); app.submit(applicationPackage).deploy(); - setZoneInRotation("rotation-fqdn-2", ZoneId.from("prod", "us-west-1")); - setZoneInRotation("rotation-fqdn-2", ZoneId.from("prod", "us-east-3")); - setZoneInRotation("rotation-fqdn-1", ZoneId.from("prod", "eu-west-1")); - // GET global rotation status without specifying endpointId fails tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET) .userIdentity(USER_ID), @@ -1817,11 +1811,6 @@ public class ApplicationApiTest extends ControllerContainerTest { } } - private void setZoneInRotation(String rotationName, ZoneId zone) { - tester.serviceRegistry().globalRoutingServiceMock().setStatus(rotationName, zone, com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus.IN); - //new RotationStatusUpdater(tester.controller(), Duration.ofDays(1)).run(); - } - private void updateContactInformation() { Contact contact = new Contact(URI.create("www.contacts.tld/1234"), URI.create("www.properties.tld/1234"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json index eb508b2459e..4955c549b4b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-with-routing-policy-legacy.json @@ -23,14 +23,6 @@ }, { "cluster": "default", - "tls": false, - "url": "http://instance1.application1.tenant1.us-west-1.prod.vespa.yahooapis.com:4080/", - "scope": "zone", - "routingMethod": "shared", - "legacy": true - }, - { - "cluster": "default", "tls": true, "url": "https://instance1--application1--tenant1.us-west-1.prod.vespa.yahooapis.com:4443/", "scope": "zone", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index d538728f7da..ca0db13b3d1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -33,7 +33,7 @@ public class DeploymentApiTest extends ControllerContainerTest { public void testDeploymentApi() { ContainerTester tester = new ContainerTester(container, responseFiles); DeploymentTester deploymentTester = new DeploymentTester(new ControllerTester(tester)); - Version version = Version.fromString("5.0"); + Version version = Version.fromString("4.9"); deploymentTester.controllerTester().upgradeSystem(version); ApplicationPackage multiInstancePackage = new ApplicationPackageBuilder() .instances("i1,i2") @@ -42,8 +42,20 @@ public class DeploymentApiTest extends ControllerContainerTest { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-west-1") .build(); + ApplicationPackage emptyPackage = new ApplicationPackageBuilder().instances("default") + .allow(ValidationId.deploymentRemoval) + .build(); - // 3 applications deploy on current system version + // Deploy application without any declared jobs on the oldest version. + var oldAppWithoutDeployment = deploymentTester.newDeploymentContext("tenant4", "application4", "default"); + oldAppWithoutDeployment.submit().failDeployment(JobType.systemTest); + oldAppWithoutDeployment.submit(emptyPackage); + + // System upgrades to 5.0 for the other applications. + version = Version.fromString("5.0"); + deploymentTester.controllerTester().upgradeSystem(version); + + // 3 applications deploy on current system version. var failingApp = deploymentTester.newDeploymentContext("tenant1", "application1", "default"); var productionApp = deploymentTester.newDeploymentContext("tenant2", "application2", "i1"); var otherProductionApp = deploymentTester.newDeploymentContext("tenant2", "application2", "i2"); diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 0bada8160be..01ca3590b0f 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -18,7 +18,7 @@ endfunction() function(setup_vespa_default_build_settings_rhel_8) message("-- Setting up default build settings for rhel 8") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "12" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_centos_7) diff --git a/defaults/pom.xml b/defaults/pom.xml index 41cf386946b..86872b6d700 100644 --- a/defaults/pom.xml +++ b/defaults/pom.xml @@ -61,12 +61,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> diff --git a/dist/STLExtras.h.diff b/dist/STLExtras.h.diff new file mode 100644 index 00000000000..40f6a2a12ba --- /dev/null +++ b/dist/STLExtras.h.diff @@ -0,0 +1,20 @@ +--- STLExtras.h.orig 2021-01-28 01:34:01.000000000 +0100 ++++ STLExtras.h 2021-03-03 22:18:46.028992086 +0100 +@@ -1820,7 +1820,7 @@ + result_pair(std::size_t Index, IterOfRange<R> Iter) + : Index(Index), Iter(Iter) {} + +- result_pair<R>(const result_pair<R> &Other) ++ result_pair(const result_pair<R> &Other) + : Index(Other.Index), Iter(Other.Iter) {} + result_pair<R> &operator=(const result_pair<R> &Other) { + Index = Other.Index; +@@ -1870,7 +1870,7 @@ + return Result.Iter == RHS.Result.Iter; + } + +- enumerator_iter<R>(const enumerator_iter<R> &Other) : Result(Other.Result) {} ++ enumerator_iter(const enumerator_iter<R> &Other) : Result(Other.Result) {} + enumerator_iter<R> &operator=(const enumerator_iter<R> &Other) { + Result = Other.Result; + return *this; diff --git a/dist/vespa.spec b/dist/vespa.spec index 31eafe51ae2..ab7bb6acb2d 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -50,11 +50,11 @@ BuildRequires: maven %define _java_home /usr/lib/jvm/java-11-amazon-corretto.%{?_arch} BuildRequires: python3-pytest %else -BuildRequires: devtoolset-10-gcc-c++ -BuildRequires: devtoolset-10-libatomic-devel -BuildRequires: devtoolset-10-binutils +BuildRequires: devtoolset-11-gcc-c++ +BuildRequires: devtoolset-11-libatomic-devel +BuildRequires: devtoolset-11-binutils BuildRequires: rh-maven35 -%define _devtoolset_enable /opt/rh/devtoolset-10/enable +%define _devtoolset_enable /opt/rh/devtoolset-11/enable %define _rhmaven35_enable /opt/rh/rh-maven35/enable BuildRequires: python36-pytest %endif @@ -69,10 +69,10 @@ BuildRequires: gcc-toolset-11-binutils BuildRequires: gcc-toolset-11-libatomic-devel %define _devtoolset_enable /opt/rh/gcc-toolset-11/enable %else -BuildRequires: gcc-toolset-10-gcc-c++ -BuildRequires: gcc-toolset-10-binutils -BuildRequires: gcc-toolset-10-libatomic-devel -%define _devtoolset_enable /opt/rh/gcc-toolset-10/enable +BuildRequires: gcc-toolset-11-gcc-c++ +BuildRequires: gcc-toolset-11-binutils +BuildRequires: gcc-toolset-11-libatomic-devel +%define _devtoolset_enable /opt/rh/gcc-toolset-11/enable %endif BuildRequires: maven BuildRequires: pybind11-devel @@ -124,7 +124,7 @@ BuildRequires: (llvm-devel >= 13.0.0 and llvm-devel < 14) BuildRequires: (llvm-devel >= 12.0.0 and llvm-devel < 13) %endif %else -BuildRequires: (llvm-devel >= 10.0.1 and llvm-devel < 11) +BuildRequires: (llvm-devel >= 12.0.1 and llvm-devel < 13) %endif BuildRequires: vespa-boost-devel >= 1.76.0-1 BuildRequires: vespa-openssl-devel >= 1.1.1l-1 @@ -302,7 +302,7 @@ Requires: vespa-gtest = 1.11.0 %define _vespa_llvm_version 12 %endif %else -%define _vespa_llvm_version 10 +%define _vespa_llvm_version 12 %endif Requires: vespa-gtest = 1.11.0 %define _extra_link_directory %{_vespa_deps_prefix}/lib64 @@ -432,7 +432,7 @@ Requires: (llvm-libs >= 13.0.0 and llvm-libs < 14) Requires: (llvm-libs >= 12.0.0 and llvm-libs < 13) %endif %else -Requires: (llvm-libs >= 10.0.1 and llvm-libs < 11) +Requires: (llvm-libs >= 12.0.1 and llvm-libs < 13) %endif Requires: vespa-protobuf = 3.19.1 %endif @@ -549,6 +549,12 @@ nearest neighbor search used for low-level benchmarking. %setup -c -D -T %else %setup -q +%if ( 0%{?el8} || 0%{?fc34} ) && %{_vespa_llvm_version} < 13 +if grep -qs 'result_pair<R>(' /usr/include/llvm/ADT/STLExtras.h +then + patch /usr/include/llvm/ADT/STLExtras.h < dist/STLExtras.h.diff +fi +%endif echo '%{version}' > VERSION case '%{version}' in *.0) diff --git a/docprocs/pom.xml b/docprocs/pom.xml index ca4cf3af50a..24df58ad268 100644 --- a/docprocs/pom.xml +++ b/docprocs/pom.xml @@ -117,12 +117,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Werror</arg> - <arg>-Xlint:all</arg> - </compilerArgs> - </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> diff --git a/document/pom.xml b/document/pom.xml index 1bfb18767eb..3faada08553 100644 --- a/document/pom.xml +++ b/document/pom.xml @@ -136,7 +136,6 @@ <arg>-Xlint:-serial</arg> <arg>-Xlint:-rawtypes</arg> <arg>-Xlint:-unchecked</arg> - <arg>-Xlint:-cast</arg> <arg>-Werror</arg> </compilerArgs> </configuration> diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java index 21e621883fe..3ce27ce6bdb 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/ContentPolicy.java @@ -33,6 +33,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Routing policy to determine which distributor in a content cluster to send data to. @@ -74,27 +75,27 @@ public class ContentPolicy extends SlobrokPolicy { public abstract static class HostFetcher { private static class Targets { - private final List<Integer> list; - private final AtomicInteger size; + private final AtomicReference<List<Integer>> list = new AtomicReference<>(); final int total; Targets() { this(List.of(), 0); } Targets(List<Integer> list, int total) { - this.list = new CopyOnWriteArrayList<>(list); - this.size = new AtomicInteger(list.size()); + this.list.set(List.copyOf(list)); this.total = Math.max(1, total); } - Integer get(int i) { - return list.get(i); + Integer get(Random randomizer) { + List<Integer> snapshot = list.get(); + return snapshot.get(randomizer.nextInt(snapshot.size())); } - void remove(Integer v) { - size.decrementAndGet(); - list.add(null); // Avoid index out of bounds for racing getters. - list.remove(v); + synchronized void remove(Integer v) { + List<Integer> snapshot = list.get(); + if (snapshot.contains(v)) { + list.set(snapshot.stream().filter((item) -> !v.equals(item)).collect(Collectors.toList())); + } } int size() { - return size.get(); + return list.get().size(); } } @@ -119,8 +120,7 @@ public class ContentPolicy extends SlobrokPolicy { // Try to use list of random targets, if at least X % of the nodes are up while (100 * targets.size() >= requiredUpPercentageToSendToKnownGoodNodes * targets.total) { - Integer distributor = targets.get(randomizer.nextInt(targets.size())); - if (distributor == null) continue; + Integer distributor = targets.get(randomizer); String targetSpec = getTargetSpec(distributor, context); if (targetSpec != null) { context.trace(3, "Sending to random node seen up in cluster state"); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 1821c8971e7..5941ed536a8 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -27,7 +27,6 @@ import java.util.logging.Logger; public class FileDownloader implements AutoCloseable { private static final Logger log = Logger.getLogger(FileDownloader.class.getName()); - private static final Duration defaultTimeout = Duration.ofMinutes(3); private static final Duration defaultSleepBetweenRetries = Duration.ofSeconds(5); public static final File defaultDownloadDirectory = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution")); @@ -38,10 +37,6 @@ public class FileDownloader implements AutoCloseable { private final FileReferenceDownloader fileReferenceDownloader; private final Downloads downloads = new Downloads(); - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor) { - this(connectionPool, supervisor, defaultDownloadDirectory, defaultTimeout, defaultSleepBetweenRetries); - } - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout) { this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 68fc916331e..065636604d9 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -52,15 +52,8 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundDoubleFlag TLS_SIZE_FRACTION = defineDoubleFlag( - "tls-size-fraction", 0.07, - List.of("baldersheim"), "2021-12-20", "2022-02-01", - "Fraction of disk available for transaction log", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - public static final UnboundStringFlag FEED_SEQUENCER_TYPE = defineStringFlag( - "feed-sequencer-type", "LATENCY", + "feed-sequencer-type", "THROUGHPUT", List.of("baldersheim"), "2020-12-02", "2022-02-01", "Selects type of sequenced executor used for feeding in proton, valid values are LATENCY, ADAPTIVE, THROUGHPUT", "Takes effect at redeployment (requires restart)", @@ -74,7 +67,7 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag FEED_MASTER_TASK_LIMIT = defineIntFlag( - "feed-master-task-limit", 0, + "feed-master-task-limit", 1000, List.of("geirst, baldersheim"), "2021-11-18", "2022-02-01", "The task limit used by the master thread in each document db in proton. Ignored when set to 0.", "Takes effect at redeployment", @@ -159,26 +152,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundDoubleFlag DISK_BLOAT_FACTOR = defineDoubleFlag( - "disk-bloat-factor", 0.2, - List.of("baldersheim"), "2021-10-08", "2022-02-01", - "Amount of bloat allowed before compacting file", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - - public static final UnboundIntFlag DOCSTORE_COMPRESSION_LEVEL = defineIntFlag( - "docstore-compression-level", 3, - List.of("baldersheim"), "2021-10-08", "2022-02-01", - "Default compression level used for document store", - "Takes effect at redeployment", - ZONE_ID, APPLICATION_ID); - - public static final UnboundIntFlag NUM_DEPLOY_HELPER_THREADS = defineIntFlag( - "num-model-builder-threads", -1, - List.of("balder"), "2021-09-09", "2022-02-01", - "Number of threads used for speeding up building of models.", - "Takes effect on first (re)start of config server"); - public static final UnboundBooleanFlag ENABLE_FEED_BLOCK_IN_DISTRIBUTOR = defineFeatureFlag( "enable-feed-block-in-distributor", true, List.of("geirst"), "2021-01-27", "2022-01-31", @@ -222,34 +195,27 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_CONCURRENT_MERGES_PER_NODE = defineIntFlag( - "max-concurrent-merges-per-node", 128, + "max-concurrent-merges-per-node", 16, List.of("balder", "vekterli"), "2021-06-06", "2022-02-01", "Specifies max concurrent merges per content node.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundIntFlag MAX_MERGE_QUEUE_SIZE = defineIntFlag( - "max-merge-queue-size", 1024, + "max-merge-queue-size", 100, List.of("balder", "vekterli"), "2021-06-06", "2022-02-01", "Specifies max size of merge queue.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag IGNORE_MERGE_QUEUE_LIMIT = defineFeatureFlag( - "ignore-merge-queue-limit", false, + "ignore-merge-queue-limit", true, List.of("vekterli", "geirst"), "2021-10-06", "2022-03-01", "Specifies if merges that are forwarded (chained) from another content node are always " + "allowed to be enqueued even if the queue is otherwise full.", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); - public static final UnboundIntFlag LARGE_RANK_EXPRESSION_LIMIT = defineIntFlag( - "large-rank-expression-limit", 8192, - List.of("baldersheim"), "2021-06-09", "2022-02-01", - "Limit for size of rank expressions distributed by filedistribution", - "Takes effect on next internal redeployment", - APPLICATION_ID); - public static final UnboundDoubleFlag MIN_NODE_RATIO_PER_GROUP = defineDoubleFlag( "min-node-ratio-per-group", 0.0, List.of("geirst", "vekterli"), "2021-07-16", "2022-03-01", @@ -319,7 +285,7 @@ public class Flags { ); public static final UnboundIntFlag DISTRIBUTOR_MERGE_BUSY_WAIT = defineIntFlag( - "distributor-merge-busy-wait", 10, + "distributor-merge-busy-wait", 1, List.of("geirst", "vekterli"), "2021-10-04", "2022-03-01", "Number of seconds that scheduling of new merge operations in the distributor should be inhibited " + "towards a content node that has indicated merge busy", @@ -327,21 +293,21 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag DISTRIBUTOR_ENHANCED_MAINTENANCE_SCHEDULING = defineFeatureFlag( - "distributor-enhanced-maintenance-scheduling", false, + "distributor-enhanced-maintenance-scheduling", true, List.of("vekterli", "geirst"), "2021-10-14", "2022-01-31", "Enable enhanced maintenance operation scheduling semantics on the distributor", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag ASYNC_APPLY_BUCKET_DIFF = defineFeatureFlag( - "async-apply-bucket-diff", false, + "async-apply-bucket-diff", true, List.of("geirst", "vekterli"), "2021-10-22", "2022-01-31", "Whether portions of apply bucket diff handling will be performed asynchronously", "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag UNORDERED_MERGE_CHAINING = defineFeatureFlag( - "unordered-merge-chaining", false, + "unordered-merge-chaining", true, List.of("vekterli", "geirst"), "2021-11-15", "2022-03-01", "Enables the use of unordered merge chains for data merge operations", "Takes effect at redeploy", @@ -414,6 +380,21 @@ public class Flags { "Takes effect on restart of Docker container", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag ZOOKEEPER_SNAPSHOT_METHOD = defineStringFlag( + "zookeeper-snapshot-method", "", + List.of("hmusum"), "2022-01-11", "2022-02-11", + "ZooKeeper snapshot method. Valid values are '', 'gz' and 'snappy'", + "Takes effect on Docker container restart", + ZONE_ID, APPLICATION_ID, NODE_TYPE); + + public static final UnboundStringFlag PERSISTENCE_ASYNC_THROTTLING = defineStringFlag( + "persistence-async-throttling", "UNLIMITED", + List.of("vekterli"), "2022-01-12", "2022-05-01", + "Sets the throttling policy used for async persistence operations on the content nodes. " + + "Valid values: UNLIMITED, DYNAMIC", + "Triggers restart, takes effect immediately", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/fsa/pom.xml b/fsa/pom.xml index 25e1167a240..db863ba5522 100644 --- a/fsa/pom.xml +++ b/fsa/pom.xml @@ -44,16 +44,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Xlint:-fallthrough</arg> - <arg>-Xlint:-serial</arg> - <arg>-Xlint:-rawtypes</arg> - <arg>-Xlint:-unchecked</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> </plugin> <plugin> <groupId>com.yahoo.vespa</groupId> diff --git a/fsa/src/main/java/com/yahoo/fsa/FSA.java b/fsa/src/main/java/com/yahoo/fsa/FSA.java index a964b32c54a..fcc940a335c 100644 --- a/fsa/src/main/java/com/yahoo/fsa/FSA.java +++ b/fsa/src/main/java/com/yahoo/fsa/FSA.java @@ -188,10 +188,10 @@ public class FSA implements Closeable { */ public Item(FSA fsa, int state) { this.fsa = fsa; - this.string = new java.util.Stack(); + this.string = new java.util.Stack<>(); this.symbol = 0; this.state = state; - this.stack = new java.util.Stack(); + this.stack = new java.util.Stack<>(); } /** @@ -199,7 +199,7 @@ public class FSA implements Closeable { */ public Item(Item item) { this.fsa = item.fsa; - this.string = new java.util.Stack(); + this.string = new java.util.Stack<>(); for (java.util.Iterator<Byte> itr = item.string.iterator(); itr.hasNext(); ) { byte b = itr.next(); this.string.push(b); @@ -415,7 +415,7 @@ public class FSA implements Closeable { if ((mmap == null) || !mmap.isDirect()) return; try { - Class unsafeClass; + Class<?> unsafeClass; try { unsafeClass = Class.forName("sun.misc.Unsafe"); } catch (Exception ex) { @@ -468,8 +468,8 @@ public class FSA implements Closeable { * @return the loaded FSA * @throws RuntimeException if the class could not be loaded */ - public static FSA loadFromResource(String resourceFileName,Class loadingClass) { - URL fsaUrl=loadingClass.getResource(resourceFileName); + public static FSA loadFromResource(String resourceFileName, Class<?> loadingClass) { + URL fsaUrl = loadingClass.getResource(resourceFileName); if ( ! "file".equals(fsaUrl.getProtocol())) { throw new RuntimeException("Could not open non-file url '" + fsaUrl + "' as a file input stream: " + "The classloader of " + loadingClass + "' does not return file urls"); diff --git a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segmenter.java b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segmenter.java index 7c3e76996bb..4edac362131 100644 --- a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segmenter.java +++ b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segmenter.java @@ -60,7 +60,7 @@ public class Segmenter { public Segments segment(String[] tokens) { Segments segments = new Segments(tokens); - LinkedList detectors = new LinkedList(); + LinkedList<Detector> detectors = new LinkedList<>(); int i=0; @@ -68,9 +68,9 @@ public class Segmenter { while(i<tokens.length){ detectors.add(new Detector(fsa.getState(), i)); - ListIterator det_it = detectors.listIterator(); + ListIterator<Detector> det_it = detectors.listIterator(); while(det_it.hasNext()){ - Detector d = (Detector)det_it.next(); + Detector d = det_it.next(); d.state().deltaWord(tokens[i]); if(d.state().isFinal()){ segments.add(new Segment(d.index(),i+1,d.state().data().getInt(0))); diff --git a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segments.java b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segments.java index 89368e2bf8f..e3bfe956a5c 100644 --- a/fsa/src/main/java/com/yahoo/fsa/segmenter/Segments.java +++ b/fsa/src/main/java/com/yahoo/fsa/segmenter/Segments.java @@ -8,7 +8,7 @@ import java.util.LinkedList; * * @author Peter Boros */ -public class Segments extends LinkedList { +public class Segments extends LinkedList<Segment> { public final static int SEGMENTATION_WEIGHTED = 0; public final static int SEGMENTATION_WEIGHTED_BIAS10 = 1; @@ -43,10 +43,12 @@ public class Segments extends LinkedList { } } - public void add(Segment s) + @Override + public boolean add(Segment s) { - super.add(s); + var result = super.add(s); _map[s.beg()][s.end()]=super.size()-1; + return result; } private void addMissingSingles() @@ -76,8 +78,8 @@ public class Segments extends LinkedList { if(idx<0 || idx>=super.size()){ return null; } - String s = new String(_tokens[((Segment)(super.get(idx))).beg()]); - for(int i=((Segment)(super.get(idx))).beg()+1;i<((Segment)(super.get(idx))).end();i++){ + String s = new String(_tokens[super.get(idx).beg()]); + for(int i = super.get(idx).beg() + 1; i < super.get(idx).end(); i++){ s += " " + _tokens[i]; } return s; @@ -88,7 +90,7 @@ public class Segments extends LinkedList { if(idx<0 || idx>=super.size()){ return -1; } - return ((Segment)(super.get(idx))).beg(); + return super.get(idx).beg(); } public int end(int idx) @@ -96,7 +98,7 @@ public class Segments extends LinkedList { if(idx<0 || idx>=super.size()){ return -1; } - return ((Segment)(super.get(idx))).end(); + return super.get(idx).end(); } public int len(int idx) @@ -104,7 +106,7 @@ public class Segments extends LinkedList { if(idx<0 || idx>=super.size()){ return -1; } - return ((Segment)(super.get(idx))).len(); + return super.get(idx).len(); } public int conn(int idx) @@ -112,9 +114,10 @@ public class Segments extends LinkedList { if(idx<0 || idx>=super.size()){ return -1; } - return ((Segment)(super.get(idx))).conn(); + return super.get(idx).conn(); } + @SuppressWarnings("fallthrough") public Segments segmentation(int method) { Segments smnt = new Segments(_tokens); @@ -170,7 +173,7 @@ public class Segments extends LinkedList { } id = bestid; while(id!=-1){ - smnt.add(((Segment)(super.get(id)))); + smnt.add(super.get(id)); id=nextid[id]; } break; @@ -189,7 +192,7 @@ public class Segments extends LinkedList { next = i; } } - smnt.add((Segment)(super.get(bestid))); + smnt.add(super.get(bestid)); pos=next; } break; @@ -302,7 +305,7 @@ public class Segments extends LinkedList { } // add segment - smnt.add((Segment)(super.get(bestid))); + smnt.add(super.get(bestid)); // check right side if(e>end(bestid)){ diff --git a/fsa/src/main/java/com/yahoo/fsa/topicpredictor/TopicPredictor.java b/fsa/src/main/java/com/yahoo/fsa/topicpredictor/TopicPredictor.java index 7049ad5495d..52dae951165 100644 --- a/fsa/src/main/java/com/yahoo/fsa/topicpredictor/TopicPredictor.java +++ b/fsa/src/main/java/com/yahoo/fsa/topicpredictor/TopicPredictor.java @@ -59,7 +59,7 @@ public class TopicPredictor extends MetaData { * as opposed to the two-argument version. * @param segment The segment string to find (all) topics for. * @return (Linked)List of PredictedTopic objects. */ - public List getPredictedTopics(String segment) { + public List<PredictedTopic> getPredictedTopics(String segment) { return getPredictedTopics(segment, 0); } @@ -70,8 +70,8 @@ public class TopicPredictor extends MetaData { * @param segment The segment string to find topics for. * @param maxTopics The max number of topics to return, 0 for all topics * @return (Linked)List of PredictedTopic objects. */ - public List getPredictedTopics(String segment, int maxTopics) { - List predictedTopics = new LinkedList(); + public List<PredictedTopic> getPredictedTopics(String segment, int maxTopics) { + List<PredictedTopic> predictedTopics = new LinkedList<>(); int segIdx = getSegmentIndex(segment); int[][] topicArr = getTopicArray(segIdx, maxTopics); diff --git a/fsa/src/test/java/com/yahoo/fsa/test/FSADataTestCase.java b/fsa/src/test/java/com/yahoo/fsa/test/FSADataTestCase.java index 3e9efc68558..28faaea1373 100644 --- a/fsa/src/test/java/com/yahoo/fsa/test/FSADataTestCase.java +++ b/fsa/src/test/java/com/yahoo/fsa/test/FSADataTestCase.java @@ -34,6 +34,7 @@ public class FSADataTestCase { this.numExceptions = 0; this.numAsserts = 0; } + @Override public void run() { for (long i = 0; i < numRuns; ++i) { state.start(); diff --git a/fsa/src/test/java/com/yahoo/fsa/test/FSAIteratorTestCase.java b/fsa/src/test/java/com/yahoo/fsa/test/FSAIteratorTestCase.java index 645536e596b..e99998e16f2 100644 --- a/fsa/src/test/java/com/yahoo/fsa/test/FSAIteratorTestCase.java +++ b/fsa/src/test/java/com/yahoo/fsa/test/FSAIteratorTestCase.java @@ -94,7 +94,7 @@ public class FSAIteratorTestCase { @Test public void testIteratorEmpty1() { state.delta("b"); - java.util.Iterator i = fsa.iterator(state); + FSA.Iterator i = fsa.iterator(state); assertFalse(i.hasNext()); try { i.next(); @@ -107,7 +107,7 @@ public class FSAIteratorTestCase { @Test public void testIteratorEmpty2() { state.delta("daciac"); - java.util.Iterator i = fsa.iterator(state); + FSA.Iterator i = fsa.iterator(state); assertFalse(i.hasNext()); try { i.next(); @@ -119,7 +119,7 @@ public class FSAIteratorTestCase { @Test public void testIteratorRemove() { - java.util.Iterator i = fsa.iterator(state); + FSA.Iterator i = fsa.iterator(state); try { i.remove(); assertFalse(true); diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index 4a79857955a..1352220166c 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Optional; import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.Callable; @@ -480,10 +481,11 @@ public abstract class ControllerHttpClient { // Note: Much more data in response, only the interesting parts of response are included in InstanceInfo for now private static InstanceInfo toInstanceInfo(HttpResponse<byte[]> response, ApplicationId applicationId) { - Set<ZoneId> zones = new HashSet<>(); + List<ZoneDeployment> zones = new ArrayList<>(); toInspector(response).field("instances").traverse((ArrayTraverser) (___, entryObject) -> - zones.add(ZoneId.from(entryObject.field("environment").asString(), - entryObject.field("region").asString()))); + zones.add(new ZoneDeployment(ZoneId.from(entryObject.field("environment").asString(), + entryObject.field("region").asString()), + entryObject.field("url").valid() ? Optional.of(entryObject.field("url").asString()) : Optional.empty()))); return new InstanceInfo(applicationId, zones); } @@ -561,21 +563,33 @@ public abstract class ControllerHttpClient { public static class InstanceInfo { private final ApplicationId applicationId; - private final Set<ZoneId> zones; + private final List<ZoneDeployment> zones; - InstanceInfo(ApplicationId applicationId, Set<ZoneId> zones) { + InstanceInfo(ApplicationId applicationId, List<ZoneDeployment> zones) { this.applicationId = applicationId; this.zones = zones; } - public ApplicationId applicationId() { - return applicationId; - } + public ApplicationId applicationId() { return applicationId; } + + public List<ZoneDeployment> zones() { return zones; } + + } - public Set<ZoneId> zones() { - return zones; + public static class ZoneDeployment { + + private final ZoneId zone; + private final Optional<String> uri; + + public ZoneDeployment(ZoneId zone, Optional<String> uri) { + this.zone = zone; + this.uri = uri; } + public ZoneId zone() { return zone; } + + public boolean isDeployed() { return uri.isPresent(); } + } } diff --git a/linguistics/abi-spec.json b/linguistics/abi-spec.json index 31612bea983..910056286ec 100644 --- a/linguistics/abi-spec.json +++ b/linguistics/abi-spec.json @@ -13,6 +13,7 @@ "public java.lang.String languageCode()", "public boolean isCjk()", "public static com.yahoo.language.Language fromLanguageTag(java.lang.String)", + "public static com.yahoo.language.Language from(java.lang.String)", "public static com.yahoo.language.Language fromLocale(java.util.Locale)", "public static com.yahoo.language.Language fromEncoding(java.lang.String)" ], diff --git a/linguistics/src/main/java/com/yahoo/language/Language.java b/linguistics/src/main/java/com/yahoo/language/Language.java index 9f60985c119..e4ac280af9e 100644 --- a/linguistics/src/main/java/com/yahoo/language/Language.java +++ b/linguistics/src/main/java/com/yahoo/language/Language.java @@ -6,6 +6,7 @@ import com.yahoo.text.Lowercase; import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Objects; /** * @author Rich Pito @@ -529,10 +530,11 @@ public enum Language { } /** - * <p>Convenience method for calling <code>fromLocale(LocaleFactory.fromLanguageTag(languageTag))</code>.</p> + * Convenience method for calling <code>fromLocale(LocaleFactory.fromLanguageTag(languageTag))</code>. + * Returns UNKNOWN when passed null or an unknown language tag. * - * @param languageTag The language tag for which the <code>Language</code> to return. - * @return the corresponding <code>Language</code>, or {@link #UNKNOWN} if not known. + * @param languageTag the language tag for which the <code>Language</code> to return + * @return the corresponding <code>Language</code>, or {@link #UNKNOWN} if not known */ public static Language fromLanguageTag(String languageTag) { if (languageTag == null) return UNKNOWN; @@ -540,6 +542,21 @@ public enum Language { } /** + * Returns the Language from a language tag + * + * @param languageTag the language tag for which the <code>Language</code> to return, cannot be null + * @return the Language instance + * @throws IllegalArgumentException if the language tag is unknown + */ + public static Language from(String languageTag) { + Objects.requireNonNull(languageTag, "languageTag cannot be null"); + Language language = fromLocale(LocaleFactory.fromLanguageTag(languageTag)); + if ( ! languageTag.equalsIgnoreCase("unknown") && language == Language.UNKNOWN) + throw new IllegalArgumentException("Unknown language tag '" + languageTag + "'"); + return language; + } + + /** * <p>Returns the <code>Language</code> whose {@link #languageCode()} is equal to <code>locale.getLanguage()</code>, with * the following additions:</p> * <ul> diff --git a/logd/src/apps/retention/retention-enforcer.sh b/logd/src/apps/retention/retention-enforcer.sh index 6355600ee4a..24bc61e5764 100755 --- a/logd/src/apps/retention/retention-enforcer.sh +++ b/logd/src/apps/retention/retention-enforcer.sh @@ -64,6 +64,7 @@ mark_pid() { } check_pidfile() { + [ -f $PIDF ] || return 0 read pid < $PIDF [ "$pid" = $$ ] && return 0 if [ "$pid" ] && [ $pid -gt $$ ]; then @@ -105,8 +106,17 @@ maybe_collect() { process_file() { dbfile="$1" now=$(date +%s) + dbf_ts_prefix=${dbfile##*.} + dbf_ts_beg=${dbf_ts_prefix}00000 + dbf_ts_end=${dbf_ts_prefix}99999 + add=$((86400 * $RETAIN_DAYS)) + earliest_expire=$((${dbf_ts_beg} + $add)) + if [ $earliest_expire -gt $now ]; then + return 0 + fi found=0 while read timestamp logfilename; do + sleep 1 for fn in $logfilename $logfilename.*z*; do if [ -f "$fn" ]; then found=1 @@ -115,8 +125,7 @@ process_file() { done done < $dbfile if [ $found = 0 ]; then - ts=${dbfile##*.}99999 - maybe_collect "$ts" "$dbfile" + maybe_collect "${dbf_ts_end}" "$dbfile" fi } @@ -124,6 +133,7 @@ process_all() { for dbf in $DBDIR/logfiles.* ; do [ -f "$dbf" ] || continue process_file "$dbf" + sleep 1 done } @@ -139,5 +149,6 @@ mainloop() { # MAIN: prepare_stuff +sleep 600 mainloop exit 0 diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index 085978375a6..8611801b9a9 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -59,7 +59,7 @@ import java.util.logging.Logger; */ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, ReplyHandler { - private static Logger log = Logger.getLogger(MessageBus.class.getName()); + private final static Logger log = Logger.getLogger(MessageBus.class.getName()); private final AtomicBoolean destroyed = new AtomicBoolean(false); private final ProtocolRepository protocolRepository = new ProtocolRepository(); private final AtomicReference<Map<String, RoutingTable>> tablesRef = new AtomicReference<>(null); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java index c98b962f671..de614e14349 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetrieverTest.java @@ -117,7 +117,7 @@ public class ApplicationMetricsRetrieverTest { wireMockRule.stubFor(get(urlPathEqualTo(config.node(0).metricsPath())) .willReturn(aResponse() .withBody(RESPONSE) - .withFixedDelay(10))); + .withFixedDelay(1000))); ApplicationMetricsRetriever retriever = new ApplicationMetricsRetriever(config); retriever.setTaskTimeout(Duration.ofMillis(1)); @@ -134,7 +134,7 @@ public class ApplicationMetricsRetrieverTest { var delayedStub = wireMockRule.stubFor(get(urlPathEqualTo(config.node(0).metricsPath())) .willReturn(aResponse() .withBody(RESPONSE) - .withFixedDelay(10))); + .withFixedDelay(1000))); ApplicationMetricsRetriever retriever = new ApplicationMetricsRetriever(config); retriever.getMetrics(); diff --git a/model-integration/pom.xml b/model-integration/pom.xml index 47ad6375491..41139394690 100644 --- a/model-integration/pom.xml +++ b/model-integration/pom.xml @@ -71,17 +71,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Xlint:-rawtypes</arg> - <arg>-Xlint:-unchecked</arg> - <arg>-Xlint:-serial</arg> - <arg>-Xlint:-cast</arg> - <arg>-Xlint:-overloads</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> </plugin> <plugin> <groupId>com.github.os72</groupId> diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/ModelImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/ModelImporter.java index b4b21d388b5..5627327d429 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/ModelImporter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/ModelImporter.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModel; import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; @@ -100,7 +101,7 @@ public abstract class ModelImporter implements MlModelImporter { for (ImportedModel.Signature signature : model.signatures().values()) { for (String outputName : signature.outputs().values()) { try { - Optional<TensorFunction> function = importExpression(graph.get(outputName), model); + Optional<TensorFunction<Reference>> function = importExpression(graph.get(outputName), model); if (function.isEmpty()) { signature.skippedOutput(outputName, "No valid output function could be found."); } @@ -112,7 +113,7 @@ public abstract class ModelImporter implements MlModelImporter { } } - private static Optional<TensorFunction> importExpression(IntermediateOperation operation, ImportedModel model) { + private static Optional<TensorFunction<Reference>> importExpression(IntermediateOperation operation, ImportedModel model) { if (model.expressions().containsKey(operation.name())) { return operation.function(); } @@ -134,7 +135,7 @@ public abstract class ModelImporter implements MlModelImporter { operation.inputs().forEach(input -> importExpression(input, model)); } - private static Optional<TensorFunction> importConstant(IntermediateOperation operation, ImportedModel model) { + private static Optional<TensorFunction<Reference>> importConstant(IntermediateOperation operation, ImportedModel model) { String name = operation.vespaName(); if (model.hasLargeConstant(name) || model.hasSmallConstant(name)) { return operation.function(); @@ -160,7 +161,7 @@ public abstract class ModelImporter implements MlModelImporter { if (operation.function().isPresent()) { String name = operation.name(); if ( ! model.expressions().containsKey(name)) { - TensorFunction function = operation.function().get(); + TensorFunction<Reference> function = operation.function().get(); if (isSignatureOutput(model, operation)) { OrderedTensorType operationType = operation.type().get(); @@ -168,7 +169,7 @@ public abstract class ModelImporter implements MlModelImporter { if ( ! operationType.equals(standardNamingType)) { List<String> renameFrom = operationType.dimensionNames(); List<String> renameTo = standardNamingType.dimensionNames(); - function = new Rename(function, renameFrom, renameTo); + function = new Rename<Reference>(function, renameFrom, renameTo); } } @@ -196,7 +197,7 @@ public abstract class ModelImporter implements MlModelImporter { private static void importFunctionExpression(IntermediateOperation operation, ImportedModel model) { if (operation.rankingExpressionFunction().isPresent()) { - TensorFunction function = operation.rankingExpressionFunction().get(); + TensorFunction<Reference> function = operation.rankingExpressionFunction().get(); try { model.function(operation.rankingExpressionFunctionName(), new RankingExpression(operation.rankingExpressionFunctionName(), function.toString())); diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/OrderedTensorType.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/OrderedTensorType.java index 37f5ae9dd29..b77960ff3fb 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/OrderedTensorType.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/OrderedTensorType.java @@ -2,6 +2,7 @@ package ai.vespa.rankingexpression.importer; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.TensorTypeParser; diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Argument.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Argument.java index e58b5341e6b..bda2f16f9e2 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Argument.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Argument.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.rankingexpression.importer.operations; +import com.yahoo.searchlib.rankingexpression.Reference; import ai.vespa.rankingexpression.importer.OrderedTensorType; import ai.vespa.rankingexpression.importer.DimensionRenamer; import com.yahoo.tensor.evaluation.VariableTensor; @@ -26,12 +27,12 @@ public class Argument extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { - TensorFunction output = new VariableTensor(vespaName(), standardNamingType.type()); + protected TensorFunction<Reference> lazyGetFunction() { + TensorFunction<Reference> output = new VariableTensor<Reference>(vespaName(), standardNamingType.type()); if ( ! standardNamingType.equals(type)) { List<String> renameFrom = standardNamingType.dimensionNames(); List<String> renameTo = type.dimensionNames(); - output = new Rename(output, renameFrom, renameTo); + output = new Rename<Reference>(output, renameFrom, renameTo); } return output; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ConcatReduce.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ConcatReduce.java index bf10eb2457b..9484545c9c1 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ConcatReduce.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ConcatReduce.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.functions.Reduce; import com.yahoo.tensor.functions.TensorFunction; @@ -25,15 +26,15 @@ public class ConcatReduce extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputFunctionsPresent(inputs.size())) return null; - TensorFunction result = inputs.get(0).function().get(); + TensorFunction<Reference> result = inputs.get(0).function().get(); for (int i = 1; i < inputs.size(); ++i) { - TensorFunction b = inputs.get(i).function().get(); - result = new com.yahoo.tensor.functions.Concat(result, b, tmpDimensionName); + TensorFunction<Reference> b = inputs.get(i).function().get(); + result = new com.yahoo.tensor.functions.Concat<>(result, b, tmpDimensionName); } - return new com.yahoo.tensor.functions.Reduce(result, aggregator, tmpDimensionName); + return new com.yahoo.tensor.functions.Reduce<>(result, aggregator, tmpDimensionName); } @Override diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ConcatV2.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ConcatV2.java index 9f3b15cddbd..6cb810aff94 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ConcatV2.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ConcatV2.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.OrderedTensorType; import ai.vespa.rankingexpression.importer.DimensionRenamer; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.functions.TensorFunction; @@ -68,14 +69,14 @@ public class ConcatV2 extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if (!inputs.stream().map(IntermediateOperation::function).allMatch(Optional::isPresent)) { return null; } - TensorFunction result = inputs.get(0).function().get(); + TensorFunction<Reference> result = inputs.get(0).function().get(); for (int i = 1; i < inputs.size() - 1; ++i) { - TensorFunction b = inputs.get(i).function().get(); - result = new com.yahoo.tensor.functions.Concat(result, b, concatDimensionName); + TensorFunction<Reference> b = inputs.get(i).function().get(); + result = new com.yahoo.tensor.functions.Concat<>(result, b, concatDimensionName); } return result; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Const.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Const.java index 859702dec40..d68b632bf61 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Const.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Const.java @@ -35,7 +35,7 @@ public class Const extends IntermediateOperation { } @Override - public Optional<TensorFunction> function() { + public Optional<TensorFunction<Reference>> function() { if (function == null) { function = lazyGetFunction(); } @@ -43,7 +43,7 @@ public class Const extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { ExpressionNode expressionNode; if (type.type().rank() == 0 && getConstantValue().isPresent()) { expressionNode = new ConstantNode(getConstantValue().get().asDoubleValue()); diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Constant.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Constant.java index a381b2cb8a0..cdc408b3e70 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Constant.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Constant.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.functions.TensorFunction; @@ -23,7 +24,7 @@ public class Constant extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { return null; // will be added by function() since this is constant. } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ConstantOfShape.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ConstantOfShape.java index c48e5592a56..d88fc34725e 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ConstantOfShape.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ConstantOfShape.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; @@ -60,10 +61,10 @@ public class ConstantOfShape extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputTypesPresent(1)) return null; ExpressionNode valueExpr = new ConstantNode(new DoubleValue(valueToFillWith)); - TensorFunction function = Generate.bound(type.type(), wrapScalar(valueExpr)); + TensorFunction<Reference> function = Generate.bound(type.type(), wrapScalar(valueExpr)); return function; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Expand.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Expand.java index eda188b339f..6d57adbd888 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Expand.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Expand.java @@ -74,7 +74,7 @@ public class Expand extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if (!allInputFunctionsPresent(2)) return null; IntermediateOperation input = inputs.get(0); diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ExpandDims.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ExpandDims.java index 027532cd02d..83132b0669c 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ExpandDims.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/ExpandDims.java @@ -4,6 +4,7 @@ package ai.vespa.rankingexpression.importer.operations; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.GeneratorLambdaFunctionNode; @@ -65,7 +66,7 @@ public class ExpandDims extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputFunctionsPresent(2)) return null; // multiply with a generated tensor created from the reduced dimensions @@ -75,9 +76,9 @@ public class ExpandDims extends IntermediateOperation { } TensorType generatedType = typeBuilder.build(); ExpressionNode generatedExpression = new ConstantNode(new DoubleValue(1)); - Generate generatedFunction = new Generate(generatedType, + Generate<Reference> generatedFunction = new Generate<>(generatedType, new GeneratorLambdaFunctionNode(generatedType, generatedExpression).asLongListToDoubleOperator()); - return new com.yahoo.tensor.functions.Join(inputs().get(0).function().get(), generatedFunction, ScalarFunctions.multiply()); + return new com.yahoo.tensor.functions.Join<>(inputs().get(0).function().get(), generatedFunction, ScalarFunctions.multiply()); } @Override diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java index bab9c47ca9a..cd0c4da6d0f 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gather.java @@ -71,7 +71,7 @@ public class Gather extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputFunctionsPresent(2)) return null; IntermediateOperation data = inputs.get(0); diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gemm.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gemm.java index 4b3208fdeb0..1f447f2a575 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gemm.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Gemm.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import com.yahoo.searchlib.rankingexpression.rule.ArithmeticNode; import com.yahoo.searchlib.rankingexpression.rule.ArithmeticOperator; @@ -78,7 +79,7 @@ public class Gemm extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! check2or3InputsPresent()) return null; OrderedTensorType aType = inputs.get(0).type().get(); @@ -86,29 +87,29 @@ public class Gemm extends IntermediateOperation { if (aType.type().rank() != 2 || bType.type().rank() != 2) throw new IllegalArgumentException("Tensors in Gemm must have rank of exactly 2"); - Optional<TensorFunction> aFunction = inputs.get(0).function(); - Optional<TensorFunction> bFunction = inputs.get(1).function(); + Optional<TensorFunction<Reference>> aFunction = inputs.get(0).function(); + Optional<TensorFunction<Reference>> bFunction = inputs.get(1).function(); if (aFunction.isEmpty() || bFunction.isEmpty()) { return null; } String joinDimension = aType.dimensions().get(1 - transposeA).name(); - TensorFunction AxB = new com.yahoo.tensor.functions.Matmul(aFunction.get(), bFunction.get(), joinDimension); - TensorFunction alphaxAxB = new TensorFunctionNode.ExpressionTensorFunction( + TensorFunction<Reference> AxB = new com.yahoo.tensor.functions.Matmul<>(aFunction.get(), bFunction.get(), joinDimension); + TensorFunction<Reference> alphaxAxB = new TensorFunctionNode.ExpressionTensorFunction( new ArithmeticNode( new TensorFunctionNode(AxB), ArithmeticOperator.MULTIPLY, new ConstantNode(new DoubleValue(alpha)))); if (inputs.size() == 3) { - Optional<TensorFunction> cFunction = inputs.get(2).function(); - TensorFunction betaxC = new TensorFunctionNode.ExpressionTensorFunction( + Optional<TensorFunction<Reference>> cFunction = inputs.get(2).function(); + TensorFunction<Reference> betaxC = new TensorFunctionNode.ExpressionTensorFunction( new ArithmeticNode( new TensorFunctionNode(cFunction.get()), ArithmeticOperator.MULTIPLY, new ConstantNode(new DoubleValue(beta)))); - return new com.yahoo.tensor.functions.Join(alphaxAxB, betaxC, ScalarFunctions.add()); + return new com.yahoo.tensor.functions.Join<>(alphaxAxB, betaxC, ScalarFunctions.add()); } return alphaxAxB; diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Identity.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Identity.java index f096cb1e54f..ab840e708a7 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Identity.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Identity.java @@ -2,6 +2,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.functions.TensorFunction; import java.util.List; @@ -20,7 +21,7 @@ public class Identity extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if (!allInputFunctionsPresent(1)) return null; return inputs.get(0).function().orElse(null); diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java index 6ebb478715a..6378442c6d0 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/IntermediateOperation.java @@ -45,8 +45,8 @@ public abstract class IntermediateOperation { protected final List<IntermediateOperation> outputs = new ArrayList<>(); protected OrderedTensorType type; - protected TensorFunction function; - protected TensorFunction rankingExpressionFunction = null; + protected TensorFunction<Reference> function; + protected TensorFunction<Reference> rankingExpressionFunction = null; protected boolean exportAsRankingFunction = false; private boolean hasRenamedDimensions = false; @@ -65,7 +65,7 @@ public abstract class IntermediateOperation { } protected abstract OrderedTensorType lazyGetType(); - protected abstract TensorFunction lazyGetFunction(); + protected abstract TensorFunction<Reference> lazyGetFunction(); public String modelName() { return modelName; } @@ -78,14 +78,14 @@ public abstract class IntermediateOperation { } /** Returns the Vespa tensor function implementing all operations from this node with inputs */ - public Optional<TensorFunction> function() { + public Optional<TensorFunction<Reference>> function() { if (function == null) { if (isConstant()) { ExpressionNode constant = new ReferenceNode(Reference.simple("constant", vespaName())); function = new TensorFunctionNode.ExpressionTensorFunction(constant); } else if (outputs.size() > 1 || exportAsRankingFunction) { rankingExpressionFunction = lazyGetFunction(); - function = new VariableTensor(rankingExpressionFunctionName(), type.type()); + function = new VariableTensor<Reference>(rankingExpressionFunctionName(), type.type()); } else { function = lazyGetFunction(); } @@ -103,7 +103,7 @@ public abstract class IntermediateOperation { public List<IntermediateOperation> outputs() { return Collections.unmodifiableList(outputs); } /** Returns a function that should be added as a ranking expression function */ - public Optional<TensorFunction> rankingExpressionFunction() { + public Optional<TensorFunction<Reference>> rankingExpressionFunction() { return Optional.ofNullable(rankingExpressionFunction); } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Join.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Join.java index 92b5f2e743b..667641dc33a 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Join.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Join.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.OrderedTensorType; import ai.vespa.rankingexpression.importer.DimensionRenamer; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.functions.Reduce; import com.yahoo.tensor.functions.ScalarFunctions; @@ -53,7 +54,7 @@ public class Join extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputTypesPresent(2)) return null; if ( ! allInputFunctionsPresent(2)) return null; @@ -63,7 +64,7 @@ public class Join extends IntermediateOperation { if (mapOperator.isPresent()) { IntermediateOperation input = inputs.get(0); input.removeDuplicateOutputsTo(this); // avoids unnecessary function export - return new com.yahoo.tensor.functions.Map(input.function().get(), mapOperator.get()); + return new com.yahoo.tensor.functions.Map<Reference>(input.function().get(), mapOperator.get()); } } @@ -86,23 +87,23 @@ public class Join extends IntermediateOperation { } } - TensorFunction aReducedFunction = a.function().get(); + TensorFunction<Reference> aReducedFunction = a.function().get(); if (aDimensionsToReduce.size() > 0) { - aReducedFunction = new Reduce(a.function().get(), Reduce.Aggregator.sum, aDimensionsToReduce); + aReducedFunction = new Reduce<Reference>(a.function().get(), Reduce.Aggregator.sum, aDimensionsToReduce); } - TensorFunction bReducedFunction = b.function().get(); + TensorFunction<Reference> bReducedFunction = b.function().get(); if (bDimensionsToReduce.size() > 0) { - bReducedFunction = new Reduce(b.function().get(), Reduce.Aggregator.sum, bDimensionsToReduce); + bReducedFunction = new Reduce<Reference>(b.function().get(), Reduce.Aggregator.sum, bDimensionsToReduce); } // retain order of inputs if (a == inputs.get(1)) { - TensorFunction temp = bReducedFunction; + TensorFunction<Reference> temp = bReducedFunction; bReducedFunction = aReducedFunction; aReducedFunction = temp; } - return new com.yahoo.tensor.functions.Join(aReducedFunction, bReducedFunction, operator); + return new com.yahoo.tensor.functions.Join<Reference>(aReducedFunction, bReducedFunction, operator); } @Override diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Map.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Map.java index 1fd0f72f416..c9b03ba9b85 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Map.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Map.java @@ -2,6 +2,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.functions.TensorFunction; import java.util.List; @@ -26,12 +27,12 @@ public class Map extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if (!allInputFunctionsPresent(1)) { return null; } - Optional<TensorFunction> input = inputs.get(0).function(); - return new com.yahoo.tensor.functions.Map(input.get(), operator); + Optional<TensorFunction<Reference>> input = inputs.get(0).function(); + return new com.yahoo.tensor.functions.Map<>(input.get(), operator); } @Override diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java index 673df9be36b..7d64a023e27 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.EmbracedNode; @@ -58,24 +59,24 @@ public class MatMul extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputTypesPresent(2)) return null; if ( ! allInputFunctionsPresent(2)) return null; OrderedTensorType typeA = inputs.get(0).type().get(); OrderedTensorType typeB = inputs.get(1).type().get(); - TensorFunction functionA = handleBroadcasting(inputs.get(0).function().get(), typeA, typeB); - TensorFunction functionB = handleBroadcasting(inputs.get(1).function().get(), typeB, typeA); + TensorFunction<Reference> functionA = handleBroadcasting(inputs.get(0).function().get(), typeA, typeB); + TensorFunction<Reference> functionB = handleBroadcasting(inputs.get(1).function().get(), typeB, typeA); - return new com.yahoo.tensor.functions.Reduce( - new Join(functionA, functionB, ScalarFunctions.multiply()), + return new com.yahoo.tensor.functions.Reduce<Reference>( + new Join<Reference>(functionA, functionB, ScalarFunctions.multiply()), Reduce.Aggregator.sum, typeA.dimensions().get(typeA.rank() - 1).name()); } - private TensorFunction handleBroadcasting(TensorFunction tensorFunction, OrderedTensorType typeA, OrderedTensorType typeB) { - List<Slice.DimensionValue> slices = new ArrayList<>(); + private TensorFunction<Reference> handleBroadcasting(TensorFunction<Reference> tensorFunction, OrderedTensorType typeA, OrderedTensorType typeB) { + List<Slice.DimensionValue<Reference>> slices = new ArrayList<>(); for (int i = 0; i < typeA.rank() - 2; ++i) { long dimSizeA = typeA.dimensions().get(i).size().get(); String dimNameA = typeA.dimensionNames().get(i); @@ -84,11 +85,11 @@ public class MatMul extends IntermediateOperation { long dimSizeB = typeB.dimensions().get(j).size().get(); if (dimSizeB > dimSizeA && dimSizeA == 1) { ExpressionNode dimensionExpression = new EmbracedNode(new ConstantNode(DoubleValue.zero)); - slices.add(new Slice.DimensionValue(Optional.of(dimNameA), wrapScalar(dimensionExpression))); + slices.add(new Slice.DimensionValue<>(Optional.of(dimNameA), wrapScalar(dimensionExpression))); } } } - return slices.size() == 0 ? tensorFunction : new Slice(tensorFunction, slices); + return slices.size() == 0 ? tensorFunction : new Slice<>(tensorFunction, slices); } @Override diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Mean.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Mean.java index a4a47ca8ce7..fd262b2892c 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Mean.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Mean.java @@ -2,6 +2,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import ai.vespa.rankingexpression.importer.DimensionRenamer; @@ -56,12 +57,12 @@ public class Mean extends IntermediateOperation { // optimization: if keepDims and one reduce dimension that has size 1: same as identity. @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputTypesPresent(2)) return null; - TensorFunction inputFunction = inputs.get(0).function().get(); - TensorFunction output = new Reduce(inputFunction, Reduce.Aggregator.avg, reduceDimensions); + TensorFunction<Reference> inputFunction = inputs.get(0).function().get(); + TensorFunction<Reference> output = new Reduce<>(inputFunction, Reduce.Aggregator.avg, reduceDimensions); if (shouldKeepDimensions()) { // multiply with a generated tensor created from the reduced dimensions TensorType.Builder typeBuilder = new TensorType.Builder(resultValueType()); @@ -70,9 +71,9 @@ public class Mean extends IntermediateOperation { } TensorType generatedType = typeBuilder.build(); ExpressionNode generatedExpression = new ConstantNode(new DoubleValue(1)); - Generate generatedFunction = new Generate(generatedType, + Generate<Reference> generatedFunction = new Generate<>(generatedType, new GeneratorLambdaFunctionNode(generatedType, generatedExpression).asLongListToDoubleOperator()); - output = new com.yahoo.tensor.functions.Join(output, generatedFunction, ScalarFunctions.multiply()); + output = new com.yahoo.tensor.functions.Join<>(output, generatedFunction, ScalarFunctions.multiply()); } return output; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Merge.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Merge.java index f208cc97d4f..e2b5930f114 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Merge.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Merge.java @@ -2,6 +2,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.functions.TensorFunction; import java.util.List; @@ -23,7 +24,7 @@ public class Merge extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { for (IntermediateOperation operation : inputs) { if (operation.function().isPresent()) { return operation.function().get(); diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/NoOp.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/NoOp.java index 1d76fa3f0a7..d8055d548ad 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/NoOp.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/NoOp.java @@ -2,6 +2,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.functions.TensorFunction; import java.util.Collections; @@ -19,7 +20,7 @@ public class NoOp extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { return null; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxCast.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxCast.java index 7b0547be7d2..164e3dc5e11 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxCast.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxCast.java @@ -2,6 +2,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.functions.TensorFunction; import onnx.Onnx.TensorProto.DataType; @@ -30,13 +31,13 @@ public class OnnxCast extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputFunctionsPresent(1)) return null; - TensorFunction input = inputs.get(0).function().get(); + TensorFunction<Reference> input = inputs.get(0).function().get(); switch (toType) { case BOOL: - return new com.yahoo.tensor.functions.Map(input, new AsBool()); + return new com.yahoo.tensor.functions.Map<>(input, new AsBool()); case INT8: case INT16: case INT32: @@ -45,7 +46,7 @@ public class OnnxCast extends IntermediateOperation { case UINT16: case UINT32: case UINT64: - return new com.yahoo.tensor.functions.Map(input, new AsInt()); + return new com.yahoo.tensor.functions.Map<>(input, new AsInt()); case FLOAT: case DOUBLE: case FLOAT16: diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConcat.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConcat.java index 2be8fc0dc4e..97818f4c27d 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConcat.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConcat.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.functions.TensorFunction; @@ -65,14 +66,14 @@ public class OnnxConcat extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if (!inputs.stream().map(IntermediateOperation::function).allMatch(Optional::isPresent)) { return null; } - TensorFunction result = inputs.get(0).function().get(); + TensorFunction<Reference> result = inputs.get(0).function().get(); for (int i = 1; i < inputs.size(); ++i) { - TensorFunction b = inputs.get(i).function().get(); - result = new com.yahoo.tensor.functions.Concat(result, b, concatDimensionName); + TensorFunction<Reference> b = inputs.get(i).function().get(); + result = new com.yahoo.tensor.functions.Concat<>(result, b, concatDimensionName); } return result; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConstant.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConstant.java index 79123cb0380..675e18da637 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConstant.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConstant.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.TensorType; @@ -36,7 +37,7 @@ public class OnnxConstant extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { return null; // will be added by function() since this is constant. } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/PlaceholderWithDefault.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/PlaceholderWithDefault.java index 3456a24f5dd..c0f825f9092 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/PlaceholderWithDefault.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/PlaceholderWithDefault.java @@ -2,6 +2,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.functions.TensorFunction; import java.util.List; @@ -22,7 +23,7 @@ public class PlaceholderWithDefault extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if (!allInputFunctionsPresent(1)) { return null; } @@ -32,7 +33,7 @@ public class PlaceholderWithDefault extends IntermediateOperation { } @Override - public Optional<TensorFunction> rankingExpressionFunction() { + public Optional<TensorFunction<Reference>> rankingExpressionFunction() { // For now, it is much more efficient to assume we always will return // the default value, as we can prune away large parts of the expression // tree by having it calculated as a constant. If a case arises where diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Range.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Range.java index 81a9e4996b4..5c4e8cd6cd0 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Range.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Range.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import com.yahoo.searchlib.rankingexpression.rule.ArithmeticNode; import com.yahoo.searchlib.rankingexpression.rule.ArithmeticOperator; @@ -58,7 +59,7 @@ public class Range extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputTypesPresent(3)) return null; String dimensionName = type().get().dimensionNames().get(0); ExpressionNode startExpr = new ConstantNode(new DoubleValue(start)); @@ -66,7 +67,7 @@ public class Range extends IntermediateOperation { ExpressionNode dimExpr = new EmbracedNode(new ReferenceNode(dimensionName)); ExpressionNode stepExpr = new ArithmeticNode(deltaExpr, ArithmeticOperator.MULTIPLY, dimExpr); ExpressionNode addExpr = new ArithmeticNode(startExpr, ArithmeticOperator.PLUS, stepExpr); - TensorFunction function = Generate.bound(type.type(), wrapScalar(addExpr)); + TensorFunction<Reference> function = Generate.bound(type.type(), wrapScalar(addExpr)); return function; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Reduce.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Reduce.java index 8e49ce15265..b7a8a4a4e43 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Reduce.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Reduce.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; @@ -72,14 +73,14 @@ public class Reduce extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputTypesPresent(1)) return null; - TensorFunction inputFunction = inputs.get(0).function().get(); + TensorFunction<Reference> inputFunction = inputs.get(0).function().get(); if (preOperator != null) { - inputFunction = new com.yahoo.tensor.functions.Map(inputFunction, preOperator); + inputFunction = new com.yahoo.tensor.functions.Map<>(inputFunction, preOperator); } - TensorFunction output = new com.yahoo.tensor.functions.Reduce(inputFunction, aggregator, reduceDimensions); + TensorFunction<Reference> output = new com.yahoo.tensor.functions.Reduce<>(inputFunction, aggregator, reduceDimensions); if (shouldKeepDimensions()) { // multiply with a generated tensor created from the reduced dimensions TensorType.Builder typeBuilder = new TensorType.Builder(resultValueType()); @@ -88,12 +89,12 @@ public class Reduce extends IntermediateOperation { } TensorType generatedType = typeBuilder.build(); ExpressionNode generatedExpression = new ConstantNode(new DoubleValue(1)); - Generate generatedFunction = new Generate(generatedType, + Generate<Reference> generatedFunction = new Generate<>(generatedType, new GeneratorLambdaFunctionNode(generatedType, generatedExpression).asLongListToDoubleOperator()); - output = new com.yahoo.tensor.functions.Join(output, generatedFunction, ScalarFunctions.multiply()); + output = new com.yahoo.tensor.functions.Join<>(output, generatedFunction, ScalarFunctions.multiply()); } if (postOperator != null) { - output = new com.yahoo.tensor.functions.Map(output, postOperator); + output = new com.yahoo.tensor.functions.Map<>(output, postOperator); } return output; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Rename.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Rename.java index 724e49084ee..d80058dfa07 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Rename.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Rename.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.functions.TensorFunction; @@ -43,9 +44,9 @@ public class Rename extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputFunctionsPresent(1)) return null; - return new com.yahoo.tensor.functions.Rename(inputs.get(0).function().orElse(null), from, to); + return new com.yahoo.tensor.functions.Rename<>(inputs.get(0).function().orElse(null), from, to); } @Override diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Reshape.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Reshape.java index 57a43158c0d..7b675fa79af 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Reshape.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Reshape.java @@ -110,12 +110,12 @@ public class Reshape extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! inputs.stream().map(IntermediateOperation::type).allMatch(Optional::isPresent) ) return null; if ( ! inputs.stream().map(IntermediateOperation::function).allMatch(Optional::isPresent) ) return null; OrderedTensorType inputType = inputs.get(0).type().get(); - TensorFunction inputFunction = inputs.get(0).function().get(); + TensorFunction<Reference> inputFunction = inputs.get(0).function().get(); return reshape(inputFunction, inputType, type); } @@ -129,7 +129,7 @@ public class Reshape extends IntermediateOperation { return new Reshape(modelName(), name(), inputs, attributeMap); } - public TensorFunction reshape(TensorFunction inputFunction, OrderedTensorType inputType, OrderedTensorType outputType) { + public TensorFunction<Reference> reshape(TensorFunction<Reference> inputFunction, OrderedTensorType inputType, OrderedTensorType outputType) { if ( ! OrderedTensorType.tensorSize(inputType.type()).equals(OrderedTensorType.tensorSize(outputType.type()))) throw new IllegalArgumentException("New and old shape of tensor must have the same size when reshaping"); diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Select.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Select.java index a189ff9c07c..9836217866b 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Select.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Select.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.functions.ScalarFunctions; @@ -34,13 +35,13 @@ public class Select extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if (!allInputFunctionsPresent(3)) { return null; } IntermediateOperation conditionOperation = inputs().get(0); - TensorFunction a = inputs().get(1).function().get(); - TensorFunction b = inputs().get(2).function().get(); + TensorFunction<Reference> a = inputs().get(1).function().get(); + TensorFunction<Reference> b = inputs().get(2).function().get(); // Shortcut: if we know during import which tensor to select, do that directly here. if (conditionOperation.getConstantValue().isPresent()) { @@ -61,13 +62,13 @@ public class Select extends IntermediateOperation { // from 'x'. We do this by individually joining 'x' and 'y' with // 'condition', and then joining the resulting two tensors. - TensorFunction conditionFunction = conditionOperation.function().get(); - TensorFunction aCond = new com.yahoo.tensor.functions.Join(a, conditionFunction, ScalarFunctions.multiply()); - TensorFunction bCond = new com.yahoo.tensor.functions.Join(b, conditionFunction, new DoubleBinaryOperator() { + TensorFunction<Reference> conditionFunction = conditionOperation.function().get(); + TensorFunction<Reference> aCond = new com.yahoo.tensor.functions.Join<>(a, conditionFunction, ScalarFunctions.multiply()); + TensorFunction<Reference> bCond = new com.yahoo.tensor.functions.Join<>(b, conditionFunction, new DoubleBinaryOperator() { @Override public double applyAsDouble(double a, double b) { return a * (1.0 - b); } @Override public String toString() { return "f(a,b)(a * (1-b))"; } }); - return new com.yahoo.tensor.functions.Join(aCond, bCond, ScalarFunctions.add()); + return new com.yahoo.tensor.functions.Join<>(aCond, bCond, ScalarFunctions.add()); } @Override diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Shape.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Shape.java index 28e0115810a..c1cffd4243e 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Shape.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Shape.java @@ -2,6 +2,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.Tensor; @@ -28,7 +29,7 @@ public class Shape extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { return null; // will be added by function() since this is constant. } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Slice.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Slice.java index ac5d66e22c1..91b7064b19c 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Slice.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Slice.java @@ -143,7 +143,7 @@ public class Slice extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if (inputs.size() < 1 || inputs.get(0).function().isEmpty()) { return null; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Softmax.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Softmax.java index 6001bef87ed..d7060b9d440 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Softmax.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Softmax.java @@ -2,6 +2,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.functions.Join; import com.yahoo.tensor.functions.Map; import com.yahoo.tensor.functions.Reduce; @@ -34,12 +35,12 @@ public class Softmax extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputFunctionsPresent(1)) return null; List<String> reduceDimensions = reduceDimensions(); - TensorFunction input = inputs.get(0).function().get(); - TensorFunction sum = new Reduce(input, Reduce.Aggregator.sum, reduceDimensions); - TensorFunction div = new Join(input, sum, ScalarFunctions.divide()); + TensorFunction<Reference> input = inputs.get(0).function().get(); + TensorFunction<Reference> sum = new Reduce<>(input, Reduce.Aggregator.sum, reduceDimensions); + TensorFunction<Reference> div = new Join<>(input, sum, ScalarFunctions.divide()); return div; } @@ -93,13 +94,13 @@ public class Softmax extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputFunctionsPresent(1)) return null; List<String> reduceDimensions = reduceDimensions(); - TensorFunction input = inputs.get(0).function().get(); - TensorFunction max = new Reduce(input, Reduce.Aggregator.max, reduceDimensions); - TensorFunction cap = new Join(input, max, ScalarFunctions.subtract()); // to avoid overflow - TensorFunction exp = new Map(cap, ScalarFunctions.exp()); + TensorFunction<Reference> input = inputs.get(0).function().get(); + TensorFunction<Reference> max = new Reduce<>(input, Reduce.Aggregator.max, reduceDimensions); + TensorFunction<Reference> cap = new Join<>(input, max, ScalarFunctions.subtract()); // to avoid overflow + TensorFunction<Reference> exp = new Map<>(cap, ScalarFunctions.exp()); return exp; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Split.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Split.java index 2e586b38c71..6f720716adb 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Split.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Split.java @@ -84,7 +84,7 @@ public class Split extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if (!allInputFunctionsPresent(1)) return null; IntermediateOperation input = inputs.get(0); @@ -104,7 +104,7 @@ public class Split extends IntermediateOperation { com.yahoo.tensor.functions.Slice<Reference> sliceIndices = new com.yahoo.tensor.functions.Slice<>(inputIndices, dimensionValues); ExpressionNode sliceExpression = new TensorFunctionNode(sliceIndices); - TensorFunction generate = Generate.bound(type.type(), wrapScalar(sliceExpression)); + TensorFunction<Reference> generate = Generate.bound(type.type(), wrapScalar(sliceExpression)); return generate; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Squeeze.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Squeeze.java index 07110b9b966..9229d6af254 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Squeeze.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Squeeze.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.functions.Reduce; @@ -52,11 +53,11 @@ public class Squeeze extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputFunctionsPresent(1)) return null; - TensorFunction inputFunction = inputs.get(0).function().get(); - return new Reduce(inputFunction, Reduce.Aggregator.sum, squeezeDimensions); + TensorFunction<Reference> inputFunction = inputs.get(0).function().get(); + return new Reduce<>(inputFunction, Reduce.Aggregator.sum, squeezeDimensions); } @Override diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Sum.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Sum.java index b8ca114343d..902144cfea2 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Sum.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Sum.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; @@ -56,11 +57,11 @@ public class Sum extends IntermediateOperation { // optimization: if keepDims and one reduce dimension that has size 1: same as identity. @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputTypesPresent(2)) return null; - TensorFunction inputFunction = inputs.get(0).function().get(); - TensorFunction output = new Reduce(inputFunction, Reduce.Aggregator.sum, reduceDimensions); + TensorFunction<Reference> inputFunction = inputs.get(0).function().get(); + TensorFunction<Reference> output = new Reduce<>(inputFunction, Reduce.Aggregator.sum, reduceDimensions); if (shouldKeepDimensions()) { // multiply with a generated tensor created from the reduced dimensions TensorType.Builder typeBuilder = new TensorType.Builder(resultValueType()); @@ -69,9 +70,9 @@ public class Sum extends IntermediateOperation { } TensorType generatedType = typeBuilder.build(); ExpressionNode generatedExpression = new ConstantNode(new DoubleValue(1)); - Generate generatedFunction = new Generate(generatedType, + Generate<Reference> generatedFunction = new Generate<>(generatedType, new GeneratorLambdaFunctionNode(generatedType, generatedExpression).asLongListToDoubleOperator()); - output = new com.yahoo.tensor.functions.Join(output, generatedFunction, ScalarFunctions.multiply()); + output = new com.yahoo.tensor.functions.Join<>(output, generatedFunction, ScalarFunctions.multiply()); } return output; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Switch.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Switch.java index f41140075d1..502f0769350 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Switch.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Switch.java @@ -2,6 +2,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.functions.TensorFunction; import java.util.List; @@ -29,7 +30,7 @@ public class Switch extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { IntermediateOperation predicateOperation = inputs().get(1); if (!predicateOperation.getConstantValue().isPresent()) { throw new IllegalArgumentException("Switch in " + name + ": predicate must be a constant"); diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Tile.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Tile.java index 7fe5e831391..4bfab284cc2 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Tile.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Tile.java @@ -62,7 +62,7 @@ public class Tile extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if (!allInputFunctionsPresent(2)) return null; IntermediateOperation input = inputs.get(0); @@ -85,7 +85,7 @@ public class Tile extends IntermediateOperation { com.yahoo.tensor.functions.Slice<Reference> sliceIndices = new com.yahoo.tensor.functions.Slice<>(inputIndices, dimensionValues); ExpressionNode sliceExpression = new TensorFunctionNode(sliceIndices); - TensorFunction generate = Generate.bound(type.type(), wrapScalar(sliceExpression)); + TensorFunction<Reference> generate = Generate.bound(type.type(), wrapScalar(sliceExpression)); return generate; } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Transpose.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Transpose.java index add24e665e6..ef51b11884a 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Transpose.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Transpose.java @@ -2,6 +2,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.functions.TensorFunction; @@ -36,7 +37,7 @@ public class Transpose extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if (!allInputFunctionsPresent(1)) return null; return inputs.get(0).function().orElse(null); diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Unsqueeze.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Unsqueeze.java index bd3130a7cd1..a73b5a4c6ef 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Unsqueeze.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/Unsqueeze.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.operations; import ai.vespa.rankingexpression.importer.DimensionRenamer; import ai.vespa.rankingexpression.importer.OrderedTensorType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; @@ -64,7 +65,7 @@ public class Unsqueeze extends IntermediateOperation { } @Override - protected TensorFunction lazyGetFunction() { + protected TensorFunction<Reference> lazyGetFunction() { if ( ! allInputFunctionsPresent(1)) return null; // multiply with a generated tensor created from the expanded dimensions @@ -74,9 +75,9 @@ public class Unsqueeze extends IntermediateOperation { } TensorType generatedType = typeBuilder.build(); ExpressionNode generatedExpression = new ConstantNode(new DoubleValue(1)); - Generate generatedFunction = new Generate(generatedType, + Generate<Reference> generatedFunction = new Generate<>(generatedType, new GeneratorLambdaFunctionNode(generatedType, generatedExpression).asLongListToDoubleOperator()); - return new com.yahoo.tensor.functions.Join(inputs().get(0).function().get(), generatedFunction, ScalarFunctions.multiply()); + return new com.yahoo.tensor.functions.Join<>(inputs().get(0).function().get(), generatedFunction, ScalarFunctions.multiply()); } @Override diff --git a/model-integration/src/main/javacc/ModelParser.jj b/model-integration/src/main/javacc/ModelParser.jj index 9944b88a745..6f6f3508beb 100644 --- a/model-integration/src/main/javacc/ModelParser.jj +++ b/model-integration/src/main/javacc/ModelParser.jj @@ -170,7 +170,7 @@ void input() : void function() : { String name, expression, parameter; - List parameters = new ArrayList(); + List< String > parameters = new ArrayList< String >(); } { ( <FUNCTION> name = identifier() diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/OnnxOperationsTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/OnnxOperationsTestCase.java index dfc4e98d409..3ef96cdf166 100644 --- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/OnnxOperationsTestCase.java +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/onnx/OnnxOperationsTestCase.java @@ -5,6 +5,7 @@ import ai.vespa.rankingexpression.importer.IntermediateGraph; import ai.vespa.rankingexpression.importer.OrderedTensorType; import ai.vespa.rankingexpression.importer.operations.Constant; import ai.vespa.rankingexpression.importer.operations.IntermediateOperation; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.evaluation.Context; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; @@ -703,7 +704,7 @@ public class OnnxOperationsTestCase { return builder.build(); } - private TensorFunction optimizeAndRename(String opName, IntermediateOperation op) { + private TensorFunction<Reference> optimizeAndRename(String opName, IntermediateOperation op) { IntermediateGraph graph = new IntermediateGraph(modelName); graph.put(opName, op); graph.outputs(graph.defaultSignature()).put(opName, opName); @@ -717,7 +718,7 @@ public class OnnxOperationsTestCase { if ( ! operationType.equals(standardNamingType)) { List<String> renameFrom = operationType.dimensionNames(); List<String> renameTo = standardNamingType.dimensionNames(); - TensorFunction func = new Rename(new ConstantTensor(tensor), renameFrom, renameTo); + TensorFunction<Reference> func = new Rename<>(new ConstantTensor<Reference>(tensor), renameFrom, renameTo); return func.evaluate(); } return tensor; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java index 12934c23926..af167fda5c6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AddNode.java @@ -15,7 +15,7 @@ import java.util.Set; public class AddNode { public final String hostname; - public final Optional<String> id; + public final String id; public final Optional<String> parentHostname; public final Optional<String> nodeFlavor; public final Optional<FlavorOverrides> flavorOverrides; @@ -24,15 +24,15 @@ public class AddNode { public final Set<String> ipAddresses; public final Set<String> additionalIpAddresses; - public static AddNode forHost(String hostname, Optional<String> id, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { + public static AddNode forHost(String hostname, String id, String nodeFlavor, Optional<FlavorOverrides> flavorOverrides, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { return new AddNode(hostname, id, Optional.empty(), Optional.of(nodeFlavor), flavorOverrides, Optional.empty(), nodeType, ipAddresses, additionalIpAddresses); } - public static AddNode forNode(String hostname, String parentHostname, NodeResources nodeResources, NodeType nodeType, Set<String> ipAddresses) { - return new AddNode(hostname, Optional.empty(), Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), nodeType, ipAddresses, Set.of()); + public static AddNode forNode(String hostname, String id, String parentHostname, NodeResources nodeResources, NodeType nodeType, Set<String> ipAddresses) { + return new AddNode(hostname, id, Optional.of(parentHostname), Optional.empty(), Optional.empty(), Optional.of(nodeResources), nodeType, ipAddresses, Set.of()); } - private AddNode(String hostname, Optional<String> id, Optional<String> parentHostname, + private AddNode(String hostname, String id, Optional<String> parentHostname, Optional<String> nodeFlavor, Optional<FlavorOverrides> flavorOverrides, Optional<NodeResources> nodeResources, NodeType nodeType, Set<String> ipAddresses, Set<String> additionalIpAddresses) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index acb6ece6fc1..6e31e699e2c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -26,7 +26,7 @@ import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; public class NodeSpec { private final String hostname; - private final Optional<String> id; + private final String id; private final NodeState state; private final NodeType type; private final String flavor; @@ -72,7 +72,7 @@ public class NodeSpec { public NodeSpec( String hostname, - Optional<String> id, + String id, Optional<DockerImage> wantedDockerImage, Optional<DockerImage> currentDockerImage, NodeState state, @@ -148,8 +148,8 @@ public class NodeSpec { return hostname; } - /** Returns the cloud-specific ID of the host. */ - public Optional<String> id() { + /** Returns unique node ID */ + public String id() { return id; } @@ -406,7 +406,7 @@ public class NodeSpec { public static class Builder { private String hostname; - private Optional<String> id = Optional.empty(); + private String id; private NodeState state; private NodeType type; private String flavor; @@ -441,6 +441,7 @@ public class NodeSpec { public Builder(NodeSpec node) { hostname(node.hostname); + id(node.id); state(node.state); type(node.type); flavor(node.flavor); @@ -477,7 +478,7 @@ public class NodeSpec { } public Builder id(String id) { - this.id = Optional.of(id); + this.id = id; return this; } @@ -786,6 +787,7 @@ public class NodeSpec { */ public static Builder testSpec(String hostname, NodeState state) { Builder builder = new Builder() + .id(hostname) .hostname(hostname) .state(state) .type(NodeType.tenant) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index 38e725360a0..b99a3bb84d7 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -162,7 +162,7 @@ public class RealNodeRepository implements NodeRepository { return new NodeSpec( node.hostname, - Optional.ofNullable(node.id), + node.id, Optional.ofNullable(node.wantedDockerImage).map(DockerImage::fromString), Optional.ofNullable(node.currentDockerImage).map(DockerImage::fromString), nodeState, @@ -244,7 +244,7 @@ public class RealNodeRepository implements NodeRepository { private static NodeRepositoryNode nodeRepositoryNodeFromAddNode(AddNode addNode) { NodeRepositoryNode node = new NodeRepositoryNode(); - node.id = addNode.id.orElse("fake-" + addNode.hostname); + node.id = addNode.id; node.hostname = addNode.hostname; node.parentHostname = addNode.parentHostname.orElse(null); addNode.nodeFlavor.ifPresent(f -> node.flavor = f); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java index a29f6a89283..45dbfd07209 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java @@ -29,7 +29,7 @@ public class CoreCollector { private static final Pattern CORE_GENERATOR_PATH_PATTERN = Pattern.compile("^Core was generated by `(?<path>.*?)'.$"); private static final Pattern EXECFN_PATH_PATTERN = Pattern.compile("^.* execfn: '(?<path>.*?)'"); private static final Pattern FROM_PATH_PATTERN = Pattern.compile("^.* from '(?<path>.*?)'"); - static final String GDB_PATH_RHEL8 = "/opt/rh/gcc-toolset-10/root/bin/gdb"; + static final String GDB_PATH_RHEL8 = "/opt/rh/gcc-toolset-11/root/bin/gdb"; static final Map<String, Object> JAVA_HEAP_DUMP_METADATA = Map.of("bin_path", "java", "backtrace", List.of("Heap dump, no backtrace available")); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index 3d7a3b73ccd..dff72fe81f1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -79,7 +79,7 @@ public class NodeAdminImpl implements NodeAdmin { @Override public void refreshContainersToRun(Set<NodeAgentContext> nodeAgentContexts) { Map<String, NodeAgentContext> nodeAgentContextsByHostname = nodeAgentContexts.stream() - .collect(Collectors.toMap(NodeAdminImpl::nodeAgentId, Function.identity())); + .collect(Collectors.toMap(ctx -> ctx.node().id(), Function.identity())); // Stop and remove NodeAgents that should no longer be running diff(nodeAgentWithSchedulerByHostname.keySet(), nodeAgentContextsByHostname.keySet()) @@ -222,14 +222,4 @@ public class NodeAdminImpl implements NodeAdmin { NodeAgent nodeAgent = nodeAgentFactory.create(contextManager, context); return new NodeAgentWithScheduler(nodeAgent, contextManager); } - - private static String nodeAgentId(NodeAgentContext nac) { - // NodeAgentImpl has some internal state that should not be reused when the same hostname is re-allocated - // to a different application/cluster, solve this by including reservation timestamp in the key. - return nac.hostname().value() + "-" + nac.node().events().stream() - .filter(event -> "reserved".equals(event.type())) - .findFirst() - .map(event -> Long.toString(event.at().toEpochMilli())) - .orElse(""); - } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index f184deab375..9b988e9a379 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -262,11 +262,20 @@ public class NodeAgentImpl implements NodeAgent { context.log(logger, "Invoking vespa-nodectl to restart services: " + restartReason); orchestratorSuspendNode(context); + ContainerResources currentResources = existingContainer.get().resources(); + ContainerResources wantedResources = currentResources.withUnlimitedCpus(); + if ( ! warmUpDuration(context).isNegative() && ! wantedResources.equals(currentResources)) { + context.log(logger, "Updating container resources: %s -> %s", + existingContainer.get().resources().toStringCpu(), wantedResources.toStringCpu()); + containerOperations.updateContainer(context, existingContainer.get().id(), wantedResources); + } + String output = containerOperations.restartVespa(context); if (!output.isBlank()) { context.log(logger, "Restart services output: " + output); } currentRestartGeneration = context.node().wantedRestartGeneration(); + firstSuccessfulHealthCheckInstant = Optional.empty(); }); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/BadTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/BadTemplateException.java new file mode 100644 index 00000000000..65d7ebaa02d --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/BadTemplateException.java @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; + +/** + * @author hakonhall + */ +public class BadTemplateException extends TemplateException { + public BadTemplateException(Cursor location, String message) { + super(message + " at " + location.calculateLocation().lineAndColumnText()); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java new file mode 100644 index 00000000000..42e13d63c19 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Form.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +/** + * Public methods common to both Template and ListElement. + * + * @author hakonhall + */ +public interface Form { + /** Set the value of a variable, e.g. %{=color}. */ + Template set(String name, String value); + + /** Set the value of a variable and/or if-condition. */ + default Template set(String name, boolean value) { return set(name, Boolean.toString(value)); } + + default Template set(String name, int value) { return set(name, Integer.toString(value)); } + default Template set(String name, long value) { return set(name, Long.toString(value)); } + + default Template set(String name, String format, String first, String... rest) { + var args = new Object[1 + rest.length]; + args[0] = first; + System.arraycopy(rest, 0, args, 1, rest.length); + var value = String.format(format, args); + + return set(name, value); + } + + /** Add an instance of a list section after any previously added (for the given name) */ + ListElement add(String name); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java new file mode 100644 index 00000000000..4a00115cee4 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/IfSection.java @@ -0,0 +1,69 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +import java.util.Optional; + +/** + * @author hakonhall + */ +class IfSection extends Section { + private final boolean negated; + private final String name; + private final Cursor nameOffset; + private final SectionList ifSections; + private final Optional<SectionList> elseSections; + + IfSection(CursorRange range, boolean negated, String name, Cursor nameOffset, + SectionList ifSections, Optional<SectionList> elseSections) { + super(range); + this.negated = negated; + this.name = name; + this.nameOffset = nameOffset; + this.ifSections = ifSections; + this.elseSections = elseSections; + } + + String name() { return name; } + Cursor nameOffset() { return nameOffset; } + + @Override + void appendTo(StringBuilder buffer) { + Optional<String> stringValue = template().getVariableValue(name); + if (stringValue.isEmpty()) + throw new TemplateNameNotSetException(name, nameOffset); + + final boolean value; + if (stringValue.get().equals("true")) { + value = true; + } else if (stringValue.get().equals("false")) { + value = false; + } else { + throw new NotBooleanValueTemplateException(name); + } + + boolean condition = negated ? !value : value; + if (condition) { + ifSections.sections().forEach(section -> section.appendTo(buffer)); + } else if (elseSections.isPresent()) { + elseSections.get().sections().forEach(section -> section.appendTo(buffer)); + } + } + + @Override + void appendCopyTo(SectionList sectionList) { + SectionList ifSectionCopy = new SectionList(ifSections.range().start(), sectionList.templateBuilder()); + ifSections.sections().forEach(section -> section.appendCopyTo(ifSectionCopy)); + + Optional<SectionList> elseSectionCopy = elseSections.map(elseSections2 -> { + SectionList elseSectionCopy2 = new SectionList(elseSections2.range().start(), + sectionList.templateBuilder()); + elseSections2.sections().forEach(section -> section.appendCopyTo(elseSectionCopy2)); + return elseSectionCopy2; + }); + + sectionList.appendIfSection(negated, name, nameOffset, range().end(), ifSectionCopy, elseSectionCopy); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListElement.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListElement.java new file mode 100644 index 00000000000..24bb9ea6523 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListElement.java @@ -0,0 +1,17 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +/** + * @author hakonhall + */ +public class ListElement implements Form { + private final Template template; + + ListElement(Template template) { this.template = template; } + + @Override + public Template set(String name, String value) { return template.set(name, value); } + + @Override + public ListElement add(String name) { return new ListElement(template.addElement(name)); } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java new file mode 100644 index 00000000000..831dc3fe5e8 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/ListSection.java @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author hakonhall + */ +class ListSection extends Section { + private final String name; + private final Cursor nameOffset; + private final Template body; + private final List<Template> elements = new ArrayList<>(); + + ListSection(CursorRange range, String name, Cursor nameOffset, Template body) { + super(range); + this.name = name; + this.nameOffset = new Cursor(nameOffset); + this.body = body; + } + + String name() { return name; } + Cursor nameOffset() { return new Cursor(nameOffset); } + + @Override + void setTemplate(Template template) { + super.setTemplate(template); + body.setParent(template); + } + + Template add() { + Template element = body.snapshot(); + element.setParent(template()); + elements.add(element); + return element; + } + + @Override + void appendTo(StringBuilder buffer) { + elements.forEach(template -> template.appendTo(buffer)); + } + + @Override + void appendCopyTo(SectionList sectionList) { + // Optimization: Reuse body in copy, since it is only used for copying. + + ListSection newSection = sectionList.appendListSection(name, nameOffset, range().end(), body); + + elements.stream() + .map(template -> { + Template templateCopy = template.snapshot(); + templateCopy.setParent(template()); + return templateCopy; + }) + .forEach(newSection.elements::add); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java new file mode 100644 index 00000000000..c03653253af --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/LiteralSection.java @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +/** + * Represents a template literal section + * + * @see Template + * @author hakonhall + */ +class LiteralSection extends Section { + LiteralSection(CursorRange range) { + super(range); + } + + @Override + void appendTo(StringBuilder buffer) { + range().appendTo(buffer); + } + + @Override + void appendCopyTo(SectionList sectionList) { + sectionList.appendLiteralSection(range().end()); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java new file mode 100644 index 00000000000..dd92af14609 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NameAlreadyExistsTemplateException.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +/** + * @author hakonhall + */ +public class NameAlreadyExistsTemplateException extends TemplateException { + public NameAlreadyExistsTemplateException(String name, CursorRange range) { + super("Name '" + name + "' already exists in the " + describeSection(range)); + } + + public NameAlreadyExistsTemplateException(String name, Cursor firstNameLocation, + Cursor secondNameLocation) { + super("Section named '" + name + "' at " + + firstNameLocation.calculateLocation().lineAndColumnText() + + " conflicts with earlier section with the same name at " + + secondNameLocation.calculateLocation().lineAndColumnText()); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NoSuchNameTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NoSuchNameTemplateException.java new file mode 100644 index 00000000000..706d347d39d --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NoSuchNameTemplateException.java @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +/** + * @author hakonhall + */ +public class NoSuchNameTemplateException extends TemplateException { + public NoSuchNameTemplateException(CursorRange range, String name) { + super("No such element '" + name + "' in the " + describeSection(range)); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NotBooleanValueTemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NotBooleanValueTemplateException.java new file mode 100644 index 00000000000..6c6d157bb47 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/NotBooleanValueTemplateException.java @@ -0,0 +1,11 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +/** + * @author hakonhall + */ +public class NotBooleanValueTemplateException extends TemplateException { + public NotBooleanValueTemplateException(String name) { + super(name + " was set to a non-boolean value: must be true or false"); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java new file mode 100644 index 00000000000..2c52fd5c34e --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Section.java @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +import java.util.Objects; + +/** + * A section of a template text. + * + * @see Template + * @author hakonhall + */ +abstract class Section { + private final CursorRange range; + private Template template; + + protected Section(CursorRange range) { + this.range = range; + } + + void setTemplate(Template template) { this.template = template; } + + /** Guaranteed to return non-null after TemplateBuilder::build() returns. */ + protected Template template() { return Objects.requireNonNull(template); } + + protected CursorRange range() { return range; } + + abstract void appendTo(StringBuilder buffer); + + abstract void appendCopyTo(SectionList sectionList); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/SectionList.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/SectionList.java new file mode 100644 index 00000000000..066f6476bcb --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/SectionList.java @@ -0,0 +1,69 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * A mutable list of sections at the same level that can be used to build a template, e.g. the if-body. + * + * @author hakonhall + */ +class SectionList { + private final Cursor start; + private final Cursor end; + private final TemplateBuilder templateBuilder; + + private final List<Section> sections = new ArrayList<>(); + + SectionList(Cursor start, TemplateBuilder templateBuilder) { + this.start = new Cursor(start); + this.end = new Cursor(start); + this.templateBuilder = templateBuilder; + } + + CursorRange range() { return new CursorRange(start, end); } + TemplateBuilder templateBuilder() { return templateBuilder; } + List<Section> sections() { return List.copyOf(sections); } + + void appendLiteralSection(Cursor end) { + CursorRange range = verifyAndUpdateEnd(end); + var section = new LiteralSection(range); + templateBuilder.addLiteralSection(section); + sections.add(section); + } + + VariableSection appendVariableSection(String name, Cursor nameOffset, Cursor end) { + CursorRange range = verifyAndUpdateEnd(end); + var section = new VariableSection(range, name, nameOffset); + templateBuilder.addVariableSection(section); + sections.add(section); + return section; + } + + void appendIfSection(boolean negated, String name, Cursor nameOffset, Cursor end, + SectionList ifSections, Optional<SectionList> elseSections) { + CursorRange range = verifyAndUpdateEnd(end); + var section = new IfSection(range, negated, name, nameOffset, ifSections, elseSections); + templateBuilder.addIfSection(section); + sections.add(section); + } + + ListSection appendListSection(String name, Cursor nameOffset, Cursor end, Template body) { + CursorRange range = verifyAndUpdateEnd(end); + var section = new ListSection(range, name, nameOffset, body); + templateBuilder.addListSection(section); + sections.add(section); + return section; + } + + private CursorRange verifyAndUpdateEnd(Cursor newEnd) { + var range = new CursorRange(this.end, newEnd); + this.end.set(newEnd); + return range; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java new file mode 100644 index 00000000000..41e8c3e65ce --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Template.java @@ -0,0 +1,105 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * The Java representation of a template text. + * + * <p>A template is a sequence of literal text and dynamic sections defined by %{...} directives:</p> + * + * <pre> + * template: section* + * section: literal | variable | list + * literal: plain text not containing %{ + * variable: %{=id} + * if: %{if [!]id}template[%{else}template]%{end} + * list: %{list id}template%{end} + * id: a valid Java identifier + * </pre> + * + * <p>Fill the template with variable values ({@link #set(String, String) set()}, set if conditions + * ({@link #set(String, boolean)}), add list elements ({@link #add(String) add()}, etc, and finally + * render it as a String ({@link #render()}).</p> + * + * <p>To reuse a template, create the template and work on snapshots of that ({@link #snapshot()}).</p> + * + * @author hakonhall + */ +public class Template implements Form { + private Template parent = null; + private final CursorRange range; + private final List<Section> sections; + + private final Map<String, String> values = new HashMap<>(); + private final Map<String, ListSection> lists; + + public static Template at(Path path) { return at(path, new TemplateDescriptor()); } + public static Template at(Path path, TemplateDescriptor descriptor) { + String content = new UnixPath(path).readUtf8File(); + return Template.from(content, descriptor); + } + + public static Template from(String text) { return from(text, new TemplateDescriptor()); } + public static Template from(String text, TemplateDescriptor descriptor) { + return TemplateParser.parse(text, descriptor).template(); + } + + Template(CursorRange range, List<Section> sections, Map<String, ListSection> lists) { + this.range = new CursorRange(range); + this.sections = List.copyOf(sections); + this.lists = Map.copyOf(lists); + } + + /** Set the value of a variable, e.g. %{=color}. */ + @Override + public Template set(String name, String value) { + values.put(name, value); + return this; + } + + @Override + public ListElement add(String name) { return new ListElement(addElement(name)); } + + public String render() { + var buffer = new StringBuilder((int) (range.length() * 1.2 + 128)); + appendTo(buffer); + return buffer.toString(); + } + + public void appendTo(StringBuilder buffer) { sections.forEach(section -> section.appendTo(buffer)); } + + /** Returns a deep copy of this. No changes to this affects the returned template, and vice versa. */ + public Template snapshot() { + var builder = new TemplateBuilder(range.start()); + sections.forEach(section -> section.appendCopyTo(builder.topLevelSectionList())); + Template template = builder.build(); + values.forEach(template::set); + return template; + } + + /** Must be called (if there is a parent) before any other method. */ + void setParent(Template parent) { this.parent = parent; } + + Template addElement(String name) { + var section = lists.get(name); + if (section == null) { + throw new NoSuchNameTemplateException(range, name); + } + return section.add(); + } + + Optional<String> getVariableValue(String name) { + String value = values.get(name); + if (value != null) return Optional.of(value); + if (parent != null) return parent.getVariableValue(name); + return Optional.empty(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java new file mode 100644 index 00000000000..8041a17fe74 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateBuilder.java @@ -0,0 +1,81 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author hakonhall + */ +class TemplateBuilder { + /** The top-level section list in this template. */ + private final SectionList sectionList; + private final List<Section> allSections = new ArrayList<>(); + private final Map<String, VariableSection> sampleVariables = new HashMap<>(); + private final Map<String, IfSection> sampleIfSections = new HashMap<>(); + private final Map<String, ListSection> lists = new HashMap<>(); + + TemplateBuilder(Cursor start) { + this.sectionList = new SectionList(start, this); + } + + SectionList topLevelSectionList() { return sectionList; } + + void addLiteralSection(LiteralSection section) { + allSections.add(section); + } + + void addVariableSection(VariableSection section) { + // It's OK if the same name is used in an if-directive (as long as the value is boolean, + // determined when set on a template). + + ListSection existing = lists.get(section.name()); + if (existing != null) + throw new NameAlreadyExistsTemplateException(section.name(), existing.nameOffset(), + section.nameOffset()); + + sampleVariables.put(section.name(), section); + allSections.add(section); + } + + void addIfSection(IfSection section) { + // It's OK if the same name is used in a variable section (as long as the value is boolean, + // determined when set on a template). + + ListSection list = lists.get(section.name()); + if (list != null) + throw new NameAlreadyExistsTemplateException(section.name(), list.nameOffset(), + section.nameOffset()); + + sampleIfSections.put(section.name(), section); + allSections.add(section); + } + + void addListSection(ListSection section) { + VariableSection variableSection = sampleVariables.get(section.name()); + if (variableSection != null) + throw new NameAlreadyExistsTemplateException(section.name(), variableSection.nameOffset(), + section.nameOffset()); + + IfSection ifSection = sampleIfSections.get(section.name()); + if (ifSection != null) + throw new NameAlreadyExistsTemplateException(section.name(), ifSection.nameOffset(), + section.nameOffset()); + + ListSection previous = lists.put(section.name(), section); + if (previous != null) + throw new NameAlreadyExistsTemplateException(section.name(), previous.nameOffset(), + section.nameOffset()); + allSections.add(section); + } + + Template build() { + var template = new Template(sectionList.range(), sectionList.sections(), lists); + allSections.forEach(section -> section.setTemplate(template)); + return template; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateDescriptor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateDescriptor.java new file mode 100644 index 00000000000..e2609d12cef --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateDescriptor.java @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +/** + * Specifies the how to interpret a template text. + * + * @author hakonhall + */ +public class TemplateDescriptor { + + private String startDelimiter = "%{"; + private String endDelimiter = "}"; + private boolean removeNewline = true; + + public TemplateDescriptor() {} + + public TemplateDescriptor(TemplateDescriptor that) { + this.startDelimiter = that.startDelimiter; + this.endDelimiter = that.endDelimiter; + this.removeNewline = that.removeNewline; + } + + /** Use these delimiters instead of the standard "%{" and "}" to start and end a template directive. */ + public TemplateDescriptor setDelimiters(String startDelimiter, String endDelimiter) { + this.startDelimiter = Token.verifyDelimiter(startDelimiter); + this.endDelimiter = Token.verifyDelimiter(endDelimiter); + return this; + } + + /** + * Whether to remove a newline that immediately follows a non-variable directive. The opposite + * effect can be achieved by preceding the end delimiter with a "-" char, e.g. %{if foo-}. + */ + public TemplateDescriptor setRemoveNewline(boolean removeNewline) { + this.removeNewline = removeNewline; + return this; + } + + public String startDelimiter() { return startDelimiter; } + public String endDelimiter() { return endDelimiter; } + public boolean removeNewline() { return removeNewline; } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateException.java new file mode 100644 index 00000000000..f231583de52 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateException.java @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +/** + * @author hakonhall + */ +public class TemplateException extends RuntimeException { + public TemplateException(String message) { super(message); } + + protected static String describeSection(CursorRange range) { + var startLocation = range.start().calculateLocation(); + var endLocation = range.end().calculateLocation(); + return "template section starting at line " + startLocation.line() + " and column " + startLocation.column() + + ", and ending at line " + endLocation.line() + " and column " + endLocation.column(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateNameNotSetException.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateNameNotSetException.java new file mode 100644 index 00000000000..d65c2a7c4d6 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateNameNotSetException.java @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; + +/** + * @author hakonhall + */ +public class TemplateNameNotSetException extends TemplateException { + public TemplateNameNotSetException(String name, Cursor nameOffset) { + super("Variable at " + nameOffset.calculateLocation().lineAndColumnText() + " has not been set: " + name); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java new file mode 100644 index 00000000000..c2202dea4a0 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateParser.java @@ -0,0 +1,161 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; + +import java.util.EnumSet; +import java.util.Optional; + +/** + * Parses a template String, see {@link Template} for details. + * + * @author hakonhall + */ +class TemplateParser { + private final TemplateDescriptor descriptor; + private final Cursor start; + private final Cursor current; + private final TemplateBuilder templateBuilder; + + static TemplateParser parse(String text, TemplateDescriptor descriptor) { + return parse(new TemplateDescriptor(descriptor), new Cursor(text), EnumSet.of(Sentinel.EOT)); + } + + private static TemplateParser parse(TemplateDescriptor descriptor, Cursor start, EnumSet<Sentinel> sentinel) { + var parser = new TemplateParser(descriptor, start); + parser.parse(parser.templateBuilder.topLevelSectionList(), sentinel); + return parser; + } + + private enum Sentinel { ELSE, END, EOT } + + private TemplateParser(TemplateDescriptor descriptor, Cursor start) { + this.descriptor = descriptor; + this.start = new Cursor(start); + this.current = new Cursor(start); + this.templateBuilder = new TemplateBuilder(start); + } + + Template template() { return templateBuilder.build(); } + + private Sentinel parse(SectionList sectionList, EnumSet<Sentinel> sentinels) { + do { + current.advanceTo(descriptor.startDelimiter()); + if (!current.equals(start)) { + sectionList.appendLiteralSection(current); + } + + if (current.eot()) { + if (!sentinels.contains(Sentinel.EOT)) { + throw new BadTemplateException(current, + "Missing end directive for section started at " + + start.calculateLocation().lineAndColumnText()); + } + return Sentinel.EOT; + } + + Optional<Sentinel> sentinel = parseSection(sectionList, sentinels); + if (sentinel.isPresent()) return sentinel.get(); + } while (true); + } + + private Optional<Sentinel> parseSection(SectionList sectionList, EnumSet<Sentinel> sentinels) { + current.skip(descriptor.startDelimiter()); + + if (current.skip(Token.VARIABLE_DIRECTIVE_CHAR)) { + parseVariableSection(sectionList); + } else { + var startOfType = new Cursor(current); + String type = skipId().orElseThrow(() -> new BadTemplateException(current, "Missing section name")); + + switch (type) { + case "else": + if (!sentinels.contains(Sentinel.ELSE)) + throw new BadTemplateException(startOfType, "Extraneous 'else'"); + parseEndDirective(); + return Optional.of(Sentinel.ELSE); + case "end": + if (!sentinels.contains(Sentinel.END)) + throw new BadTemplateException(startOfType, "Extraneous 'end'"); + parseEndDirective(); + return Optional.of(Sentinel.END); + case "if": + parseIfSection(sectionList); + break; + case "list": + parseListSection(sectionList); + break; + default: + throw new BadTemplateException(startOfType, "Unknown section '" + type + "'"); + } + } + + return Optional.empty(); + } + + private void parseVariableSection(SectionList sectionList) { + var nameStart = new Cursor(current); + String name = parseId(); + parseEndDelimiter(false); + sectionList.appendVariableSection(name, nameStart, current); + } + + private void parseEndDirective() { + parseEndDelimiter(true); + } + + private void parseListSection(SectionList sectionList) { + skipRequiredWhitespaces(); + var startOfName = new Cursor(current); + String name = parseId(); + parseEndDelimiter(true); + + TemplateParser bodyParser = parse(descriptor, current, EnumSet.of(Sentinel.END)); + current.set(bodyParser.current); + + sectionList.appendListSection(name, startOfName, current, bodyParser.templateBuilder.build()); + } + + private void parseIfSection(SectionList sectionList) { + skipRequiredWhitespaces(); + boolean negated = current.skip(Token.NEGATE_CHAR); + current.skipWhitespaces(); + var startOfName = new Cursor(current); + String name = parseId(); + parseEndDelimiter(true); + + SectionList ifSectionList = new SectionList(current, templateBuilder); + Sentinel ifSentinel = parse(ifSectionList, EnumSet.of(Sentinel.ELSE, Sentinel.END)); + + Optional<SectionList> elseSectionList = Optional.empty(); + if (ifSentinel == Sentinel.ELSE) { + elseSectionList = Optional.of(new SectionList(current, templateBuilder)); + parse(elseSectionList.get(), EnumSet.of(Sentinel.END)); + } + + sectionList.appendIfSection(negated, name, startOfName, current, ifSectionList, elseSectionList); + } + + private void skipRequiredWhitespaces() { + if (!current.skipWhitespaces()) { + throw new BadTemplateException(current, "Expected whitespace"); + } + } + + private String parseId() { + return skipId().orElseThrow(() -> new BadTemplateException(current, "Expected identifier")); + } + + private Optional<String> skipId() { return Token.skipId(current); } + + private void parseEndDelimiter(boolean allowSkipNewline) { + boolean removeNewlineCharPresent = current.skip(Token.REMOVE_NEWLINE_CHAR); + + if (!current.skip(descriptor.endDelimiter())) + throw new BadTemplateException(current, "Expected section end (" + descriptor.endDelimiter() + ")"); + + // The presence of the remove-newline-char means the opposite behavior is wanted. + if (allowSkipNewline && (removeNewlineCharPresent != descriptor.removeNewline())) + current.skip('\n'); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Token.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Token.java new file mode 100644 index 00000000000..a83dab72025 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/Token.java @@ -0,0 +1,60 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +import java.util.Optional; + +/** + * @author hakonhall + */ +class Token { + static final char NEGATE_CHAR = '!'; + static final char REMOVE_NEWLINE_CHAR = '-'; + static final char VARIABLE_DIRECTIVE_CHAR = '='; + + static Optional<String> skipId(Cursor cursor) { + if (cursor.eot() || !isIdStart(cursor.getChar())) return Optional.empty(); + + Cursor start = new Cursor(cursor); + cursor.increment(); + + while (!cursor.eot() && isIdPart(cursor.getChar())) + cursor.increment(); + + return Optional.of(new CursorRange(start, cursor).string()); + } + + /** A delimiter either starts a directive (e.g. %{) or ends it (e.g. }). */ + static String verifyDelimiter(String delimiter) { + if (!isAsciiToken(delimiter)) { + throw new IllegalArgumentException("Invalid delimiter: '" + delimiter + "'"); + } + return delimiter; + } + + /** Returns true for a non-empty string with only ASCII token characters. */ + private static boolean isAsciiToken(String string) { + if (string.isEmpty()) return false; + for (char c : string.toCharArray()) { + if (!isAsciiTokenChar(c)) return false; + } + return true; + } + + /** Returns true if char is a printable ASCII character except space (isgraph(3)). */ + private static boolean isAsciiTokenChar(char c) { + // 0x1F unit separator + // 0x20 space + // 0x21 ! + // ... + // 0x7E ~ + // 0x7F del + return 0x20 < c && c < 0x7F; + } + + // Our identifiers are equivalent to a Java identifiers. + private static boolean isIdStart(char c) { return Character.isJavaIdentifierStart(c); } + private static boolean isIdPart(char c) { return Character.isJavaIdentifierPart(c); } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java new file mode 100644 index 00000000000..6a7bec2e485 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/VariableSection.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.vespa.hosted.node.admin.task.util.text.Cursor; +import com.yahoo.vespa.hosted.node.admin.task.util.text.CursorRange; + +/** + * Represents a template variable section + * + * @see Template + * @author hakonhall + */ +class VariableSection extends Section { + private final String name; + private final Cursor nameOffset; + + VariableSection(CursorRange range, String name, Cursor nameOffset) { + super(range); + this.name = name; + this.nameOffset = nameOffset; + } + + String name() { return name; } + Cursor nameOffset() { return new Cursor(nameOffset); } + + @Override + void appendTo(StringBuilder buffer) { + String value = template().getVariableValue(name) + .orElseThrow(() -> new TemplateNameNotSetException(name, nameOffset)); + buffer.append(value); + } + + @Override + void appendCopyTo(SectionList sectionList) { + sectionList.appendVariableSection(name, nameOffset, range().end()); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/package-info.java new file mode 100644 index 00000000000..5bb8f656305 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/template/package-info.java @@ -0,0 +1,5 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/Cursor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/Cursor.java new file mode 100644 index 00000000000..2fc3f8bac60 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/Cursor.java @@ -0,0 +1,165 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.text; + +import java.util.Objects; + +/** + * Cursor is a mutable offset into a fixed String, and useful for String parsing. + * + * @author hakonhall + */ +// @Mutable +public class Cursor { + private final String text; + private int offset; + private TextLocation locationCache; + + /** Creates a pointer to the first char of {@code text}, which is EOT if {@code text} is empty. */ + public Cursor(String text) { this(text, 0, new TextLocation()); } + + public Cursor(Cursor that) { this(that.text, that.offset, that.locationCache); } + + private Cursor(String text, int offset, TextLocation location) { + this.text = Objects.requireNonNull(text); + this.offset = offset; + this.locationCache = Objects.requireNonNull(location); + } + + /** Returns the substring of {@code text} starting at {@link #offset()} (to EOT). */ + @Override + public String toString() { return text.substring(offset); } + + public String fullText() { return text; } + public int offset() { return offset; } + public boolean bot() { return offset == 0; } + public boolean eot() { return offset == text.length(); } + public boolean startsWith(char c) { return offset < text.length() && text.charAt(offset) == c; } + public boolean startsWith(String prefix) { return text.startsWith(prefix, offset); } + + /** @throws IndexOutOfBoundsException if {@link #eot()}. */ + public char getChar() { return text.charAt(offset); } + + /** The number of chars between pointer and EOT. */ + public int length() { return text.length() - offset; } + + /** Calculate the current text location in O(length(text)). */ + public TextLocation calculateLocation() { + if (offset < locationCache.offset()) { + locationCache = new TextLocation(); + } else if (offset == locationCache.offset()) { + return locationCache; + } + + int lineIndex = locationCache.lineIndex(); + int columnIndex = locationCache.columnIndex(); + for (int i = locationCache.offset(); i < offset; ++i) { + if (text.charAt(i) == '\n') { + ++lineIndex; + columnIndex = 0; + } else { + ++columnIndex; + } + } + + locationCache = new TextLocation(offset, lineIndex, columnIndex); + return locationCache; + } + + public void set(Cursor that) { + if (that.text != text) { + throw new IllegalArgumentException("'that' doesn't refer to the same text"); + } + + this.offset = that.offset; + } + + /** Advance substring.length() if this startsWith the substring, returning true if so. */ + public boolean skip(String substring) { + if (startsWith(substring)) { + offset += substring.length(); + return true; + } else { + return false; + } + } + + public boolean skip(char c) { + if (startsWith(c)) { + ++offset; + return true; + } else { + return false; + } + } + + /** If the current char is a whitespace, skip it and return true. */ + public boolean skipWhitespace() { + if (!eot() && Character.isWhitespace(getChar())) { + ++offset; + return true; + } else { + return false; + } + } + + /** Returns true if at least one whitespace was skipped. */ + public boolean skipWhitespaces() { + if (skipWhitespace()) { + while (skipWhitespace()) + ++offset; + return true; + } else { + return false; + } + } + + /** Return false if eot(), otherwise advance to the next char and return true. */ + public boolean increment() { + if (eot()) return false; + ++offset; + return true; + } + + /** + * Advance {@code distance} chars until bot() or eot() is reached (distance may be negative), + * and return true if this cursor moved the full distance. + */ + public boolean advance(int distance) { + int newOffset = offset + distance; + if (newOffset < 0) { + this.offset = 0; + return false; + } else if (newOffset > text.length()) { + this.offset = text.length(); + return false; + } else { + this.offset = newOffset; + return true; + } + } + + /** Advance pointer until start of needle is found (and return true), or EOT is reached (and return false). */ + public boolean advanceTo(String needle) { + int index = text.indexOf(needle, offset); + if (index == -1) { + offset = text.length(); + return false; // and eot() is true + } else { + offset = index; + return true; // and eot() is false + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Cursor cursor = (Cursor) o; + return offset == cursor.offset && text.equals(cursor.text); + } + + @Override + public int hashCode() { + return Objects.hash(text, offset); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/CursorRange.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/CursorRange.java new file mode 100644 index 00000000000..23ac69ccee2 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/CursorRange.java @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.text; + +/** + * A start- and end- offset in an underlying String. + * + * @author hakonhall + */ +public class CursorRange { + private final Cursor start; + private final Cursor end; + + @SuppressWarnings("StringEquality") + public CursorRange(Cursor start, Cursor end) { + if (start.fullText() != end.fullText()) { + throw new IllegalArgumentException("start and end points to different texts"); + } + + if (start.offset() > end.offset()) { + throw new IllegalArgumentException("start offset " + start.offset() + + " is beyond end offset " + end.offset()); + } + + this.start = new Cursor(start); + this.end = new Cursor(end); + } + + public CursorRange(CursorRange that) { + this.start = new Cursor(that.start); + this.end = new Cursor(that.end); + } + + public Cursor start() { return new Cursor(start); } + public Cursor end() { return new Cursor(end); } + public int length() { return end.offset() - start.offset(); } + public String string() { return start.fullText().substring(start.offset(), end.offset()); } + public void appendTo(StringBuilder buffer) { buffer.append(start.fullText(), start.offset(), end.offset()); } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/TextLocation.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/TextLocation.java new file mode 100644 index 00000000000..32441c842b0 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/text/TextLocation.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.text; + +/** + * The location within an implied multi-line String. + * + * @author hakonhall + */ +//@Immutable +public class TextLocation { + private final int offset; + private final int lineIndex; + private final int columnIndex; + + public TextLocation() { this(0, 0, 0); } + + public TextLocation(int offset, int lineIndex, int columnIndex) { + this.offset = offset; + this.lineIndex = lineIndex; + this.columnIndex = columnIndex; + } + + public int offset() { return offset; } + public int lineIndex() { return lineIndex; } + public int line() { return lineIndex + 1; } + public int columnIndex() { return columnIndex; } + public int column() { return columnIndex + 1; } + + public String lineAndColumnText() { return "line " + line() + " and column " + column(); } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 5d15d4353e2..3256b16a6c5 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -161,20 +161,20 @@ public class RealNodeRepositoryTest { @Test public void testAddNodes() { AddNode host = AddNode.forHost("host123.domain.tld", - Optional.of("id1"), + "id1", "default", Optional.of(FlavorOverrides.ofDisk(123)), NodeType.confighost, Set.of("::1"), Set.of("::2", "::3")); NodeResources nodeResources = new NodeResources(1, 2, 3, 4, NodeResources.DiskSpeed.slow, NodeResources.StorageType.local); - AddNode node = AddNode.forNode("host123-1.domain.tld", "host123.domain.tld", nodeResources, NodeType.config, Set.of("::2", "::3")); + AddNode node = AddNode.forNode("host123-1.domain.tld", "id1", "host123.domain.tld", nodeResources, NodeType.config, Set.of("::2", "::3")); assertFalse(nodeRepositoryApi.getOptionalNode("host123.domain.tld").isPresent()); nodeRepositoryApi.addNodes(List.of(host, node)); NodeSpec hostSpec = nodeRepositoryApi.getOptionalNode("host123.domain.tld").orElseThrow(); - assertEquals("id1", hostSpec.id().orElseThrow()); + assertEquals("id1", hostSpec.id()); assertEquals("default", hostSpec.flavor()); assertEquals(123, hostSpec.diskGb(), 0); assertEquals(NodeType.confighost, hostSpec.type()); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java index b4ff7a0ceae..5c334837040 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java @@ -86,7 +86,7 @@ public class CoreCollectorTest { fail("Expected not to be able to get bin path"); } catch (RuntimeException e) { assertEquals("Failed to extract binary path from GDB, result: exit status 1, output 'Error 123', command: " + - "[/bin/sh, -c, /opt/rh/gcc-toolset-10/root/bin/gdb -n -batch -core /tmp/core.1234 | grep '^Core was generated by']", e.getMessage()); + "[/bin/sh, -c, /opt/rh/gcc-toolset-11/root/bin/gdb -n -batch -core /tmp/core.1234 | grep '^Core was generated by']", e.getMessage()); } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 13f101be6e8..d87a60e4a44 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -699,6 +699,42 @@ public class NodeAgentImplTest { inOrder.verify(orchestrator, never()).resume(any(String.class)); } + @Test + public void uncaps_and_caps_cpu_for_services_restart() { + NodeSpec.Builder specBuilder = nodeBuilder(NodeState.active) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) + .wantedRestartGeneration(2).currentRestartGeneration(1); + + NodeAgentContext context = createContext(specBuilder.build()); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); + mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true); + + InOrder inOrder = inOrder(orchestrator, containerOperations); + + nodeAgent.converge(context); + inOrder.verify(orchestrator, times(1)).suspend(eq(hostName)); + inOrder.verify(containerOperations, times(1)).updateContainer(eq(context), eq(containerId), eq(ContainerResources.from(0, 0, 16))); + inOrder.verify(containerOperations, times(1)).restartVespa(eq(context)); + + mockGetContainer(dockerImage, ContainerResources.from(0, 0, 16), true); + doNothing().when(healthChecker).verifyHealth(any()); + try { + nodeAgent.doConverge(context); + fail("Expected to fail due to warm up period not yet done"); + } catch (ConvergenceException e) { + assertEquals("Refusing to resume until warm up period ends (in PT30S)", e.getMessage()); + } + inOrder.verify(orchestrator, never()).resume(any()); + inOrder.verify(orchestrator, never()).suspend(any()); + inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); + + + clock.advance(Duration.ofSeconds(31)); + nodeAgent.doConverge(context); + inOrder.verify(orchestrator, times(1)).resume(eq(hostName)); + } + private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) { NodeSpec.Builder nodeBuilder = nodeBuilder(nodeState) .type(NodeType.tenant) diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java new file mode 100644 index 00000000000..e010b9780c6 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/template/TemplateTest.java @@ -0,0 +1,153 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.template; + +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author hakonhall + */ +public class TemplateTest { + @Test + void verifyNewlineRemoval() { + Template template = Template.from("a%{list a}\n" + + "b%{end}\n" + + "c%{list c-}\n" + + "d%{end-}\n" + + "e\n", + new TemplateDescriptor().setRemoveNewline(false)); + template.add("a"); + template.add("c"); + + assertEquals("a\n" + + "b\n" + + "cde\n", + template.render()); + } + + @Test + void verifyIfSection() { + Template template = Template.from("Hello%{if cond} world%{end}!"); + assertEquals("Hello world!", template.snapshot().set("cond", true).render()); + assertEquals("Hello!", template.snapshot().set("cond", false).render()); + } + + @Test + void verifyComplexIfSection() { + Template template = Template.from("%{if cond}\n" + + "var: %{=varname}\n" + + "if: %{if !inner}inner is false%{end-}\n" + + "list: %{list formname}element%{end-}\n" + + "%{end}\n"); + + assertEquals("", template.snapshot().set("cond", false).render()); + + assertEquals("var: varvalue\n" + + "if: \n" + + "list: \n", + template.snapshot() + .set("cond", true) + .set("varname", "varvalue") + .set("inner", true) + .render()); + + Template template2 = template.snapshot() + .set("cond", true) + .set("varname", "varvalue") + .set("inner", false); + template2.add("formname"); + + assertEquals("var: varvalue\n" + + "if: inner is false\n" + + "list: element\n", template2.render()); + } + + @Test + void verifyElse() { + var template = Template.from("%{if cond}\n" + + "if body\n" + + "%{else}\n" + + "else body\n" + + "%{end}\n"); + assertEquals("if body\n", template.snapshot().set("cond", true).render()); + assertEquals("else body\n", template.snapshot().set("cond", false).render()); + } + + @Test + void verifySnapshotPreservesList() { + var template = Template.from("%{list foo}hello %{=area}%{end}"); + template.add("foo") + .set("area", "world"); + + assertEquals("hello world", template.render()); + assertEquals("hello world", template.snapshot().render()); + + Template snapshot = template.snapshot(); + snapshot.add("foo") + .set("area", "Norway"); + assertEquals("hello worldhello Norway", snapshot.render()); + } + + @Test + void verifyVariableSection() { + Template template = getTemplate("template1.tmp"); + template.set("varname", "varvalue"); + assertEquals("variable section 'varvalue'\n" + + "end of text\n", template.render()); + } + + @Test + void verifySimpleListSection() { + Template template = getTemplate("template1.tmp"); + template.set("varname", "varvalue") + .add("listname") + .set("varname", "different varvalue") + .set("varname2", "varvalue2"); + assertEquals("variable section 'varvalue'\n" + + "same variable section 'different varvalue'\n" + + "different variable section 'varvalue2'\n" + + "between ends\n" + + "end of text\n", template.render()); + } + + @Test + void verifyNestedListSection() { + Template template = getTemplate("template2.tmp"); + ListElement A0 = template.add("listA"); + ListElement A0B0 = A0.add("listB"); + ListElement A0B1 = A0.add("listB"); + + ListElement A1 = template.add("listA"); + ListElement A1B0 = A1.add("listB"); + assertEquals("body A\n" + + "body B\n" + + "body B\n" + + "body A\n" + + "body B\n", + template.render()); + } + + @Test + void verifyVariableReferences() { + Template template = getTemplate("template3.tmp"); + template.set("varname", "varvalue") + .set("innerVarSetAtTop", "val2"); + template.add("l"); + template.add("l") + .set("varname", "varvalue2"); + assertEquals("varvalue\n" + + "varvalue\n" + + "inner varvalue\n" + + "val2\n" + + "inner varvalue2\n" + + "val2\n", + template.render()); + } + + private Template getTemplate(String filename) { + return Template.at(Path.of("src/test/resources/" + filename)); + } +} diff --git a/node-admin/src/test/resources/template1.tmp b/node-admin/src/test/resources/template1.tmp new file mode 100644 index 00000000000..d53c875a0f3 --- /dev/null +++ b/node-admin/src/test/resources/template1.tmp @@ -0,0 +1,10 @@ +variable section '%{=varname}' +%{list listname} +same variable section '%{=varname}' +different variable section '%{=varname2}' +%{list innerlistname} +inner form text +%{end} +between ends +%{end} +end of text diff --git a/node-admin/src/test/resources/template2.tmp b/node-admin/src/test/resources/template2.tmp new file mode 100644 index 00000000000..d36cb4a4a48 --- /dev/null +++ b/node-admin/src/test/resources/template2.tmp @@ -0,0 +1,4 @@ +%{list listA}body A +%{list listB}body B +%{end} +%{end} diff --git a/node-admin/src/test/resources/template3.tmp b/node-admin/src/test/resources/template3.tmp new file mode 100644 index 00000000000..27566e72a9d --- /dev/null +++ b/node-admin/src/test/resources/template3.tmp @@ -0,0 +1,6 @@ +%{=varname} +%{=varname} +%{list l} +inner %{=varname} +%{=innerVarSetAtTop} +%{end} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 4d63863a917..3db68a27234 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; /** @@ -63,7 +64,7 @@ public final class Node implements Nodelike { /** Creates a node builder in the initial state (reserved) */ public static Node.Builder reserve(Set<String> ipAddresses, String hostname, String parentHostname, NodeResources resources, NodeType type) { - return new Node.Builder("fake-" + hostname, hostname, new Flavor(resources), State.reserved, type) + return new Node.Builder(UUID.randomUUID().toString(), hostname, new Flavor(resources), State.reserved, type) .ipConfig(IP.Config.ofEmptyPool(ipAddresses)) .parentHostname(parentHostname); } @@ -140,7 +141,7 @@ public final class Node implements Nodelike { * * - OpenStack: UUID * - AWS: Instance ID - * - Linux containers: fake-[hostname] + * - Linux containers: UUID */ public String id() { return id; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 80c192f8353..abacaf8e264 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -72,7 +72,8 @@ public class Autoscaler { if (scaledIn(clusterModel.scalingDuration(), cluster)) return Advice.dontScale(Status.waiting, - "Won't autoscale now: Less than " + clusterModel.scalingDuration() + " since last resource change"); + "Won't autoscale now: Less than " + clusterModel.scalingDuration() + + " since last resource change"); if (clusterModel.nodeTimeseries().measurementsPerNode() < minimumMeasurementsPerNode(clusterModel.scalingDuration())) return Advice.none(Status.waiting, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index 3c26eef41d9..b7a5c1e7fe7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -25,7 +25,7 @@ public class ClusterModel { private static final Logger log = Logger.getLogger(ClusterModel.class.getName()); /** Containers typically use more cpu right after generation change, so discard those metrics */ - public static final Duration warmupDuration = Duration.ofSeconds(90); + public static final Duration warmupDuration = Duration.ofMinutes(5); private static final Duration currentLoadDuration = Duration.ofMinutes(5); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java index dd35eb7b54d..5ad4ef2e263 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java @@ -31,7 +31,7 @@ public class ClusterNodesTimeseries { // If none can be detected we assume the node is new/was down. // If either this is the case, or there is a generation change, we ignore // the first warmupWindow metrics - var timeseries = db.getNodeTimeseries(period.plus(warmupDuration.multipliedBy(8)), clusterNodes); + var timeseries = db.getNodeTimeseries(period.plus(warmupDuration.multipliedBy(4)), clusterNodes); if (cluster.lastScalingEvent().isPresent()) { long currentGeneration = cluster.lastScalingEvent().get().generation(); timeseries = keepCurrentGenerationAfterWarmup(timeseries, currentGeneration); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java index c58b08cb3b5..4a5f8972e11 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java @@ -85,7 +85,7 @@ public class NodeTimeseries { if (snapshot.generation() < 0) return true; // Content nodes do not yet send generation if (snapshot.generation() < currentGeneration) return false; if (generationChange.isEmpty()) return true; - return ! snapshot.at().isBefore(generationChange.get().plus(warmupDuration.multipliedBy(2))); + return ! snapshot.at().isBefore(generationChange.get().plus(warmupDuration)); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 874e9cbe2c5..ca14a1be4c4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -369,9 +369,7 @@ public class MetricsReporter extends NodeRepositoryMaintainer { static Map<String, String> dimensions(ApplicationId application, ClusterSpec.Id cluster) { Map<String, String> dimensions = new HashMap<>(dimensions(application)); - //TODO: Remove "clusterId" once internal aggregation uses "clusterid" dimensions.put("clusterid", cluster.value()); - dimensions.put("clusterId", cluster.value()); return dimensions; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 480fd72967e..7f57ec219ae 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeMutex; import com.yahoo.vespa.hosted.provision.maintenance.NodeFailer; -import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; +import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.hosted.provision.persistence.CuratorDatabaseClient; import java.time.Clock; @@ -591,7 +591,7 @@ public class Nodes { * @return the nodes in their new state */ public List<Node> restartActive(Predicate<Node> filter) { - return restart(StateFilter.from(Node.State.active).and(filter)); + return restart(NodeFilter.in(Set.of(Node.State.active)).and(filter)); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java new file mode 100644 index 00000000000..a65ec30264f --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/NodeFilter.java @@ -0,0 +1,63 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node.filter; + + +import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.hosted.provision.Node; + +import java.util.EnumSet; +import java.util.Objects; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * A filter for nodes, matching by state. This should be the top-most filter so that the node-repository can determine + * which node states to read before testing additional filters. + * + * @author mpolden + */ +public class NodeFilter implements Predicate<Node> { + + private final Set<Node.State> states; + private final Predicate<Node> filter; + + private NodeFilter(Set<Node.State> states, Predicate<Node> filter) { + this.states = Objects.requireNonNull(states); + this.filter = Objects.requireNonNull(filter); + } + + @Override + public boolean test(Node node) { + return states.contains(node.state()) && filter.test(node); + } + + /** The node states to match */ + public Set<Node.State> states() { + return states; + } + + /** Returns a copy of this that matches with given filter */ + public NodeFilter matching(Predicate<Node> filter) { + return new NodeFilter(states, filter); + } + + /** Returns a node filter which matches a comma or space-separated list of states */ + public static NodeFilter in(String states, boolean includeDeprovisioned) { + if (states == null) { + return NodeFilter.in(includeDeprovisioned + ? EnumSet.allOf(Node.State.class) + : EnumSet.complementOf(EnumSet.of(Node.State.deprovisioned))); + } + return NodeFilter.in(StringUtilities.split(states).stream() + .map(Node.State::valueOf) + .collect(Collectors.toSet())); + } + + /** Returns a node filter matching given states */ + public static NodeFilter in(Set<Node.State> states) { + return new NodeFilter(states, (ignored) -> true); + } + + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java deleted file mode 100644 index 9e3928ecbe5..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/filter/StateFilter.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.node.filter; - -import com.yahoo.text.StringUtilities; -import com.yahoo.vespa.hosted.provision.Node; - -import java.util.EnumSet; -import java.util.Objects; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -/** - * A node filter which filters on node states. - * - * @author bratseth - */ -public class StateFilter { - - private StateFilter() {} - - private static Predicate<Node> makePredicate(EnumSet<Node.State> states) { - Objects.requireNonNull(states, "state cannot be null, use an empty set"); - return node -> states.contains(node.state()); - } - - /** Returns a copy of the given filter which only matches for the given state */ - public static Predicate<Node> from(Node.State state) { - return makePredicate(EnumSet.of(state)); - } - - /** Returns a node filter which matches a comma or space-separated list of states */ - public static Predicate<Node> from(String states, boolean includeDeprovisioned) { - if (states == null) { - return makePredicate(includeDeprovisioned ? - EnumSet.allOf(Node.State.class) : EnumSet.complementOf(EnumSet.of(Node.State.deprovisioned))); - } - - return makePredicate(StringUtilities.split(states).stream() - .map(Node.State::valueOf) - .collect(Collectors.toCollection(() -> EnumSet.noneOf(Node.State.class)))); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 7d03c14f172..543972a9cb3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -254,10 +254,10 @@ public class NodeSerializer { // ---------------- Deserialization -------------------------------------------------- public Node fromJson(Node.State state, byte[] data) { - var key = Hashing.sipHash24().newHasher() - .putString(state.name(), StandardCharsets.UTF_8) - .putBytes(data).hash() - .asLong(); + long key = Hashing.sipHash24().newHasher() + .putString(state.name(), StandardCharsets.UTF_8) + .putBytes(data).hash() + .asLong(); try { return cache.get(key, () -> nodeFromSlime(state, SlimeUtils.jsonToSlime(data).get())); } catch (ExecutionException e) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 4852f0f8269..922c8bc8e20 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -5,27 +5,28 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.TrustStoreItem; +import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.vespa.orchestrator.status.HostInfo; import com.yahoo.vespa.orchestrator.status.HostStatus; import java.net.URI; +import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.function.Function; -import java.util.function.Predicate; /** * @author bratseth @@ -41,7 +42,7 @@ class NodesResponse extends SlimeJsonResponse { /** The parent url of nodes */ private final String nodeParentUrl; - private final Predicate<Node> filter; + private final NodeFilter filter; private final boolean recursive; private final Function<HostName, Optional<HostInfo>> orchestrator; private final NodeRepository nodeRepository; @@ -57,9 +58,9 @@ class NodesResponse extends SlimeJsonResponse { Cursor root = slime.setObject(); switch (responseType) { - case nodeList: nodesToSlime(root); break; + case nodeList: nodesToSlime(filter.states(), root); break; case stateList : statesToSlime(root); break; - case nodesInStateList: nodesToSlime(NodeSerializer.stateFrom(lastElement(parentUrl)), root); break; + case nodesInStateList: nodesToSlime(Set.of(NodeSerializer.stateFrom(lastElement(parentUrl))), root); break; case singleNode : nodeToSlime(lastElement(parentUrl), root); break; default: throw new IllegalArgumentException(); } @@ -87,23 +88,23 @@ class NodesResponse extends SlimeJsonResponse { private void toSlime(Node.State state, Cursor object) { object.setString("url", parentUrl + NodeSerializer.toString(state)); if (recursive) - nodesToSlime(state, object); + nodesToSlime(Set.of(state), object); } - /** Outputs the nodes in the given state to a node array */ - private void nodesToSlime(Node.State state, Cursor parentObject) { + /** Outputs the nodes in the given states to a node array */ + private void nodesToSlime(Set<Node.State> statesToRead, Cursor parentObject) { Cursor nodeArray = parentObject.setArray("nodes"); - for (NodeType type : NodeType.values()) - toSlime(nodeRepository.nodes().list(state).nodeType(type).asList(), nodeArray); - } - - /** Outputs all the nodes to a node array */ - private void nodesToSlime(Cursor parentObject) { - Cursor nodeArray = parentObject.setArray("nodes"); - toSlime(nodeRepository.nodes().list().asList(), nodeArray); + boolean sortByNodeType = statesToRead.size() == 1; + statesToRead.stream().sorted().forEach(state -> { + NodeList nodes = nodeRepository.nodes().list(state); + if (sortByNodeType) { + nodes = nodes.sortedBy(Comparator.comparing(Node::type)); + } + toSlime(nodes, nodeArray); + }); } - private void toSlime(List<Node> nodes, Cursor array) { + private void toSlime(NodeList nodes, Cursor array) { for (Node node : nodes) { if ( ! filter.test(node)) continue; toSlime(node, recursive, array.addObject()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 67bb69b6191..15e1061f5e1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -35,11 +35,11 @@ import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; +import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeHostFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeOsVersionFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeTypeFilter; import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter; -import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; import com.yahoo.vespa.hosted.provision.restapi.NodesResponse.ResponseType; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.yolean.Exceptions; @@ -56,7 +56,6 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.function.Function; -import java.util.function.Predicate; import java.util.logging.Level; import java.util.stream.Collectors; @@ -322,16 +321,17 @@ public class NodesV2ApiHandler extends LoggingRequestHandler { return NodeSerializer.typeFrom(object.asString()); } - public static Predicate<Node> toNodeFilter(HttpRequest request) { - return NodeHostFilter.from(HostFilter.from(request.getProperty("hostname"), - request.getProperty("flavor"), - request.getProperty("clusterType"), - request.getProperty("clusterId"))) - .and(ApplicationFilter.from(request.getProperty("application"))) - .and(StateFilter.from(request.getProperty("state"), request.getBooleanProperty("includeDeprovisioned"))) - .and(NodeTypeFilter.from(request.getProperty("type"))) - .and(ParentHostFilter.from(request.getProperty("parentHost"))) - .and(NodeOsVersionFilter.from(request.getProperty("osVersion"))); + public static NodeFilter toNodeFilter(HttpRequest request) { + return NodeFilter.in(request.getProperty("state"), + request.getBooleanProperty("includeDeprovisioned")) + .matching(NodeHostFilter.from(HostFilter.from(request.getProperty("hostname"), + request.getProperty("flavor"), + request.getProperty("clusterType"), + request.getProperty("clusterId"))) + .and(ApplicationFilter.from(request.getProperty("application"))) + .and(NodeTypeFilter.from(request.getProperty("type"))) + .and(ParentHostFilter.from(request.getProperty("parentHost"))) + .and(NodeOsVersionFilter.from(request.getProperty("osVersion")))); } private static boolean isPatchOverride(HttpRequest request) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json index c9784c7e610..b5efc69d6db 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json @@ -1,6 +1,6 @@ { "url": "http://localhost:8080/nodes/v2/node/test-node-pool-102-2", - "id": "fake-test-node-pool-102-2", + "id": "(ignore)", "state": "active", "type": "tenant", "hostname": "test-node-pool-102-2", diff --git a/parent/pom.xml b/parent/pom.xml index 36d682e7895..8afebd281c9 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -87,8 +87,6 @@ <arg>-Xlint:-serial</arg> <arg>-Xlint:-try</arg> <arg>-Xlint:-processing</arg> - <arg>-Xlint:-varargs</arg> - <arg>-Xlint:-options</arg> <arg>-Werror</arg> </compilerArgs> </configuration> @@ -351,13 +349,6 @@ <artifactId>maven-javadoc-plugin</artifactId> <executions> <execution> - <id>generate-javadoc</id> - <phase>package</phase> - <goals> - <goal>javadoc-no-fork</goal> - </goals> - </execution> - <execution> <id>attach-javadocs</id> <goals> <goal>jar</goal> @@ -483,7 +474,7 @@ <dependency> <groupId>com.ibm.icu</groupId> <artifactId>icu4j</artifactId> - <version>57.1</version> + <version>57.2</version> </dependency> <dependency> <groupId>com.infradna.tool</groupId> @@ -798,7 +789,7 @@ <dependency> <groupId>org.hdrhistogram</groupId> <artifactId>HdrHistogram</artifactId> - <version>2.1.8</version> + <version>${hdrhistogram.version}</version> </dependency> <dependency> <groupId>org.junit.jupiter</groupId> @@ -896,6 +887,7 @@ <commons.codec.version>1.15</commons.codec.version> <commons.math3.version>3.6.1</commons.math3.version> <gson.version>2.8.9</gson.version> + <hdrhistogram.version>2.1.12</hdrhistogram.version> <jna.version>5.9.0</jna.version> <junit.version>5.8.1</junit.version> <maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version> @@ -915,7 +907,7 @@ <mockito.version>4.0.0</mockito.version> <onnxruntime.version>1.8.0</onnxruntime.version> <prometheus.client.version>0.6.0</prometheus.client.version> - <protobuf.version>3.11.4</protobuf.version> + <protobuf.version>3.19.2</protobuf.version> <spifly.version>1.3.3</spifly.version> <surefire.version>2.22.2</surefire.version> <zookeeper.client.version>3.7.0</zookeeper.client.version> diff --git a/persistence/src/tests/dummyimpl/dummypersistence_test.cpp b/persistence/src/tests/dummyimpl/dummypersistence_test.cpp index 21cf7cb9ae3..3fa1a8a9b8d 100644 --- a/persistence/src/tests/dummyimpl/dummypersistence_test.cpp +++ b/persistence/src/tests/dummyimpl/dummypersistence_test.cpp @@ -9,7 +9,6 @@ #include <vespa/vdslib/state/clusterstate.h> #include <vespa/config-stor-distribution.h> - using namespace storage::spi; using namespace storage; using document::test::makeBucketSpace; @@ -20,14 +19,14 @@ namespace { struct Fixture { BucketContent content; - void insert(DocumentId id, Timestamp timestamp, int meta_flags) { - content.insert(DocEntry::UP(new DocEntry(timestamp, meta_flags, id))); + void insert(DocumentId id, Timestamp timestamp, DocumentMetaEnum meta_flags) { + content.insert(DocEntry::create(timestamp, meta_flags, id)); } Fixture() { - insert(DocumentId("id:ns:type::test:3"), Timestamp(3), NONE); - insert(DocumentId("id:ns:type::test:1"), Timestamp(1), NONE); - insert(DocumentId("id:ns:type::test:2"), Timestamp(2), NONE); + insert(DocumentId("id:ns:type::test:3"), Timestamp(3), DocumentMetaEnum::NONE); + insert(DocumentId("id:ns:type::test:1"), Timestamp(1), DocumentMetaEnum::NONE); + insert(DocumentId("id:ns:type::test:2"), Timestamp(2), DocumentMetaEnum::NONE); } }; @@ -64,13 +63,13 @@ TEST_F("require that BucketContent can provide bucket info", Fixture) { uint32_t lastChecksum = 0; EXPECT_NOT_EQUAL(lastChecksum, f.content.getBucketInfo().getChecksum()); lastChecksum = f.content.getBucketInfo().getChecksum(); - f.insert(DocumentId("id:ns:type::test:3"), Timestamp(4), NONE); + f.insert(DocumentId("id:ns:type::test:3"), Timestamp(4), DocumentMetaEnum::NONE); EXPECT_NOT_EQUAL(lastChecksum, f.content.getBucketInfo().getChecksum()); lastChecksum = f.content.getBucketInfo().getChecksum(); - f.insert(DocumentId("id:ns:type::test:2"), Timestamp(5), REMOVE_ENTRY); + f.insert(DocumentId("id:ns:type::test:2"), Timestamp(5), DocumentMetaEnum::REMOVE_ENTRY); EXPECT_NOT_EQUAL(lastChecksum, f.content.getBucketInfo().getChecksum()); - f.insert(DocumentId("id:ns:type::test:1"), Timestamp(6), REMOVE_ENTRY); - f.insert(DocumentId("id:ns:type::test:3"), Timestamp(7), REMOVE_ENTRY); + f.insert(DocumentId("id:ns:type::test:1"), Timestamp(6), DocumentMetaEnum::REMOVE_ENTRY); + f.insert(DocumentId("id:ns:type::test:3"), Timestamp(7), DocumentMetaEnum::REMOVE_ENTRY); EXPECT_EQUAL(0u, f.content.getBucketInfo().getChecksum()); } diff --git a/persistence/src/tests/spi/clusterstatetest.cpp b/persistence/src/tests/spi/clusterstatetest.cpp index ac67903244f..2186d408791 100644 --- a/persistence/src/tests/spi/clusterstatetest.cpp +++ b/persistence/src/tests/spi/clusterstatetest.cpp @@ -5,10 +5,12 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/config-stor-distribution.h> +#include <vespa/document/base/testdocman.h> #include <gtest/gtest.h> using storage::spi::test::makeSpiBucket; using vespalib::Trinary; +using document::GlobalId; namespace storage::spi { @@ -260,4 +262,57 @@ TEST(ClusterStateTest, node_maintenance_state_is_set_independent_of_bucket_space EXPECT_FALSE(node_marked_as_maintenance_in_state("distributor:3 storage:3 .0.s:m", d, 0, false)); } +TEST(DocEntryTest, test_basics) { + EXPECT_EQ(24, sizeof(DocEntry)); +} + +TEST(DocEntryTest, test_meta_only) { + DocEntry::UP e = DocEntry::create(Timestamp(9), DocumentMetaEnum::NONE); + EXPECT_EQ(9, e->getTimestamp()); + EXPECT_FALSE(e->isRemove()); + EXPECT_EQ(24, e->getSize()); + EXPECT_EQ(nullptr, e->getDocument()); + EXPECT_EQ(nullptr, e->getDocumentId()); + EXPECT_EQ("", e->getDocumentType()); + EXPECT_EQ(GlobalId(), e->getGid()); + + DocEntry::UP r = DocEntry::create(Timestamp(666), DocumentMetaEnum::REMOVE_ENTRY); + EXPECT_EQ(666, r->getTimestamp()); + EXPECT_TRUE(r->isRemove()); +} + +TEST(DocEntryTest, test_docid_only) { + DocEntry::UP e = DocEntry::create(Timestamp(9), DocumentMetaEnum::NONE, DocumentId("id:test:test::1")); + EXPECT_EQ(9, e->getTimestamp()); + EXPECT_FALSE(e->isRemove()); + EXPECT_EQ(16, e->getSize()); + EXPECT_EQ(nullptr, e->getDocument()); + EXPECT_NE(nullptr, e->getDocumentId()); + EXPECT_EQ("test", e->getDocumentType()); + EXPECT_EQ(GlobalId::parse("gid(0xc4ca4238f9f9649222750be2)"), e->getGid()); +} + +TEST(DocEntryTest, test_doctype_and_gid) { + DocEntry::UP e = DocEntry::create(Timestamp(9), DocumentMetaEnum::NONE, "doc_type", GlobalId::parse("gid(0xc4cef118f9f9649222750be2)")); + EXPECT_EQ(9, e->getTimestamp()); + EXPECT_FALSE(e->isRemove()); + EXPECT_EQ(20, e->getSize()); + EXPECT_EQ(nullptr, e->getDocument()); + EXPECT_EQ(nullptr, e->getDocumentId()); + EXPECT_EQ("doc_type", e->getDocumentType()); + EXPECT_EQ(GlobalId::parse("gid(0xc4cef118f9f9649222750be2)"), e->getGid()); +} + +TEST(DocEntryTest, test_document_only) { + document::TestDocMan testDocMan; + DocEntry::UP e = DocEntry::create(Timestamp(9), testDocMan.createRandomDocument(0, 1000)); + EXPECT_EQ(9, e->getTimestamp()); + EXPECT_FALSE(e->isRemove()); + EXPECT_EQ(632, e->getSize()); + EXPECT_NE(nullptr, e->getDocument()); + EXPECT_NE(nullptr, e->getDocumentId()); + EXPECT_EQ("testdoctype1", e->getDocumentType()); + EXPECT_EQ(GlobalId::parse("gid(0x4bc7000087365609f22f1f4b)"), e->getGid()); +} + } diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index 810f2ad2356..6afdd142457 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -5,6 +5,7 @@ #include <vespa/persistence/spi/test.h> #include <vespa/persistence/spi/catchresult.h> #include <vespa/persistence/spi/resource_usage_listener.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/document/update/documentupdate.h> #include <vespa/document/update/assignvalueupdate.h> @@ -25,15 +26,17 @@ using document::BucketId; using document::BucketSpace; using document::test::makeBucketSpace; using storage::spi::test::makeSpiBucket; +using storage::spi::test::cloneDocEntry; namespace storage::spi { using PersistenceProviderUP = std::unique_ptr<PersistenceProvider>; +using DocEntryList = std::vector<DocEntry::UP>; namespace { -std::unique_ptr<PersistenceProvider> getSpi(ConformanceTest::PersistenceFactory &factory, - const document::TestDocMan &testDocMan) { +std::unique_ptr<PersistenceProvider> +getSpi(ConformanceTest::PersistenceFactory &factory, const document::TestDocMan &testDocMan) { PersistenceProviderUP result(factory.getPersistenceImplementation( testDocMan.getTypeRepoSP(), *testDocMan.getTypeConfig())); EXPECT_TRUE(!result->initialize().hasError()); @@ -123,7 +126,7 @@ struct DocAndTimestamp */ struct Chunk { - std::vector<DocEntry::UP> _entries; + DocEntryList _entries; }; struct DocEntryIndirectTimestampComparator @@ -166,7 +169,7 @@ doIterate(PersistenceProvider& spi, } size_t -getRemoveEntryCount(const std::vector<spi::DocEntry::UP>& entries) +getRemoveEntryCount(const DocEntryList& entries) { size_t ret = 0; for (size_t i = 0; i < entries.size(); ++i) { @@ -177,13 +180,13 @@ getRemoveEntryCount(const std::vector<spi::DocEntry::UP>& entries) return ret; } -std::vector<DocEntry::UP> +DocEntryList getEntriesFromChunks(const std::vector<Chunk>& chunks) { - std::vector<spi::DocEntry::UP> ret; + DocEntryList ret; for (size_t chunk = 0; chunk < chunks.size(); ++chunk) { for (size_t i = 0; i < chunks[chunk]._entries.size(); ++i) { - ret.push_back(DocEntry::UP(chunks[chunk]._entries[i]->clone())); + ret.push_back(cloneDocEntry(*chunks[chunk]._entries[i])); } } std::sort(ret.begin(), @@ -193,12 +196,12 @@ getEntriesFromChunks(const std::vector<Chunk>& chunks) } -std::vector<DocEntry::UP> +DocEntryList iterateBucket(PersistenceProvider& spi, const Bucket& bucket, IncludedVersions versions) { - std::vector<DocEntry::UP> ret; + DocEntryList ret; DocumentSelection docSel(""); Selection sel(docSel); @@ -217,7 +220,7 @@ iterateBucket(PersistenceProvider& spi, spi.iterate(iter.getIteratorId(), std::numeric_limits<int64_t>().max(), context); if (result.getErrorCode() != Result::ErrorType::NONE) { - return std::vector<DocEntry::UP>(); + return DocEntryList(); } auto list = result.steal_entries(); std::move(list.begin(), list.end(), std::back_inserter(ret)); @@ -238,8 +241,7 @@ verifyDocs(const std::vector<DocAndTimestamp>& wanted, const std::vector<Chunk>& chunks, const std::set<string>& removes = std::set<string>()) { - std::vector<DocEntry::UP> retrieved( - getEntriesFromChunks(chunks)); + DocEntryList retrieved = getEntriesFromChunks(chunks); size_t removeCount = getRemoveEntryCount(retrieved); // Ensure that we've got the correct number of puts and removes EXPECT_EQ(removes.size(), removeCount); @@ -257,15 +259,13 @@ verifyDocs(const std::vector<DocAndTimestamp>& wanted, } EXPECT_EQ(wanted[wantedIdx].timestamp, entry.getTimestamp()); size_t serSize = wanted[wantedIdx].doc->serialize().size(); - EXPECT_EQ(serSize + sizeof(DocEntry), size_t(entry.getSize())); - EXPECT_EQ(serSize, size_t(entry.getDocumentSize())); + EXPECT_EQ(serSize, size_t(entry.getSize())); ++wantedIdx; } else { // Remove-entry EXPECT_TRUE(entry.getDocumentId() != 0); size_t serSize = entry.getDocumentId()->getSerializedSize(); - EXPECT_EQ(serSize + sizeof(DocEntry), size_t(entry.getSize())); - EXPECT_EQ(serSize, size_t(entry.getDocumentSize())); + EXPECT_EQ(serSize, size_t(entry.getSize())); if (removes.find(entry.getDocumentId()->toString()) == removes.end()) { FAIL() << "Got unexpected remove entry for document id " << *entry.getDocumentId(); @@ -697,8 +697,7 @@ TEST_F(ConformanceTest, testPutDuplicate) EXPECT_EQ(1, (int)info.getDocumentCount()); EXPECT_EQ(checksum, info.getChecksum()); } - std::vector<DocEntry::UP> entries( - iterateBucket(*spi, bucket, ALL_VERSIONS)); + DocEntryList entries = iterateBucket(*spi, bucket, ALL_VERSIONS); EXPECT_EQ(size_t(1), entries.size()); } @@ -722,8 +721,7 @@ TEST_F(ConformanceTest, testRemove) EXPECT_EQ(1, (int)info.getDocumentCount()); EXPECT_TRUE(info.getChecksum() != 0); - std::vector<DocEntry::UP> entries( - iterateBucket(*spi, bucket, NEWEST_DOCUMENT_ONLY)); + DocEntryList entries = iterateBucket(*spi, bucket, NEWEST_DOCUMENT_ONLY); EXPECT_EQ(size_t(1), entries.size()); } @@ -741,15 +739,11 @@ TEST_F(ConformanceTest, testRemove) EXPECT_EQ(true, result2.wasFound()); } { - std::vector<DocEntry::UP> entries(iterateBucket(*spi, - bucket, - NEWEST_DOCUMENT_ONLY)); + DocEntryList entries = iterateBucket(*spi, bucket,NEWEST_DOCUMENT_ONLY); EXPECT_EQ(size_t(0), entries.size()); } { - std::vector<DocEntry::UP> entries(iterateBucket(*spi, - bucket, - NEWEST_DOCUMENT_OR_REMOVE)); + DocEntryList entries = iterateBucket(*spi, bucket,NEWEST_DOCUMENT_OR_REMOVE); EXPECT_EQ(size_t(1), entries.size()); } @@ -862,8 +856,7 @@ TEST_F(ConformanceTest, testRemoveMerge) // Remove entry should exist afterwards { - std::vector<DocEntry::UP> entries(iterateBucket( - *spi, bucket, ALL_VERSIONS)); + DocEntryList entries = iterateBucket(*spi, bucket, ALL_VERSIONS); EXPECT_EQ(size_t(2), entries.size()); // Timestamp-sorted by iterateBucket EXPECT_EQ(removeId, *entries.back()->getDocumentId()); @@ -889,7 +882,7 @@ TEST_F(ConformanceTest, testRemoveMerge) } // Must have new remove. We don't check for the presence of the old remove. { - std::vector<DocEntry::UP> entries(iterateBucket(*spi, bucket, ALL_VERSIONS)); + DocEntryList entries = iterateBucket(*spi, bucket, ALL_VERSIONS); EXPECT_TRUE(entries.size() >= 2); EXPECT_EQ(removeId, *entries.back()->getDocumentId()); EXPECT_EQ(Timestamp(11), entries.back()->getTimestamp()); @@ -915,7 +908,7 @@ TEST_F(ConformanceTest, testRemoveMerge) } // Must have newest remove. We don't check for the presence of the old remove. { - std::vector<DocEntry::UP> entries(iterateBucket(*spi, bucket, ALL_VERSIONS)); + DocEntryList entries = iterateBucket(*spi, bucket, ALL_VERSIONS); EXPECT_TRUE(entries.size() >= 2); EXPECT_EQ(removeId, *entries.back()->getDocumentId()); EXPECT_EQ(Timestamp(11), entries.back()->getTimestamp()); @@ -1351,7 +1344,7 @@ TEST_F(ConformanceTest, testIterateRemoves) CreateIteratorResult iter(createIterator(*spi, b, sel, NEWEST_DOCUMENT_OR_REMOVE)); std::vector<Chunk> chunks = doIterate(*spi, iter.getIteratorId(), 4_Ki); - std::vector<DocEntry::UP> entries = getEntriesFromChunks(chunks); + DocEntryList entries = getEntriesFromChunks(chunks); EXPECT_EQ(docs.size(), entries.size()); verifyDocs(nonRemovedDocs, chunks, removedDocs); diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp index 9d9f31b63a3..d947ca51f49 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.cpp @@ -9,6 +9,7 @@ #include <vespa/persistence/spi/i_resource_usage_listener.h> #include <vespa/persistence/spi/resource_usage.h> #include <vespa/persistence/spi/bucketexecutor.h> +#include <vespa/persistence/spi/test.h> #include <vespa/vespalib/util/crc.h> #include <vespa/document/fieldset/fieldsetrepo.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -24,6 +25,9 @@ using vespalib::make_string; using std::binary_search; using std::lower_bound; using document::FixedBucketSpaces; +using document::FieldSet; +using storage::spi::test::cloneDocEntry; +using storage::spi::test::equal; namespace storage::spi::dummy { @@ -162,7 +166,7 @@ BucketContent::insert(DocEntry::SP e) { auto it = lower_bound(_entries.begin(), _entries.end(), e->getTimestamp(), TimestampLess()); if (it != _entries.end()) { if (it->entry->getTimestamp() == e->getTimestamp()) { - if (*it->entry.get() == *e) { + if (equal(*it->entry, *e)) { LOG(debug, "Ignoring duplicate put entry %s", e->toString().c_str()); return; } else { @@ -431,7 +435,7 @@ DummyPersistence::putAsync(const Bucket& b, Timestamp t, Document::SP doc, Conte } } else { LOG(spam, "Inserting document %s", doc->toString(true).c_str()); - auto entry = std::make_unique<DocEntry>(t, NONE, Document::UP(doc->clone())); + auto entry = DocEntry::create(t, Document::UP(doc->clone())); (*bc)->insert(std::move(entry)); bc.reset(); onComplete->onComplete(std::make_unique<Result>()); @@ -489,7 +493,7 @@ DummyPersistence::removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentI } DocEntry::SP entry((*bc)->getEntry(id)); numRemoves += (entry.get() && !entry->isRemove()) ? 1 : 0; - auto remEntry = std::make_unique<DocEntry>(t, REMOVE_ENTRY, id); + auto remEntry = DocEntry::create(t, DocumentMetaEnum::REMOVE_ENTRY, id); if ((*bc)->hasTimestamp(t)) { (*bc)->eraseEntry(t); @@ -501,7 +505,7 @@ DummyPersistence::removeAsync(const Bucket& b, std::vector<TimeStampAndDocumentI } GetResult -DummyPersistence::get(const Bucket& b, const document::FieldSet& fieldSet, const DocumentId& did, Context&) const +DummyPersistence::get(const Bucket& b, const FieldSet& fieldSet, const DocumentId& did, Context&) const { DUMMYPERSISTENCE_VERIFY_INITIALIZED; LOG(debug, "get(%s, %s)", @@ -518,8 +522,8 @@ DummyPersistence::get(const Bucket& b, const document::FieldSet& fieldSet, const return GetResult::make_for_tombstone(entry->getTimestamp()); } else { Document::UP doc(entry->getDocument()->clone()); - if (fieldSet.getType() != document::FieldSet::Type::ALL) { - document::FieldSet::stripFields(*doc, fieldSet); + if (fieldSet.getType() != FieldSet::Type::ALL) { + FieldSet::stripFields(*doc, fieldSet); } return GetResult(std::move(doc), entry->getTimestamp()); } @@ -649,22 +653,16 @@ DummyPersistence::iterate(IteratorId id, uint64_t maxByteSize, Context& ctx) con if (currentSize != 0 && currentSize + size > maxByteSize) break; currentSize += size; if (!entry->isRemove() - && it->_fieldSet->getType() != document::FieldSet::Type::ALL) + && it->_fieldSet->getType() != FieldSet::Type::ALL) { assert(entry->getDocument()); // Create new document with only wanted fields. - Document::UP filtered( - document::FieldSet::createDocumentSubsetCopy( - *entry->getDocument(), - *it->_fieldSet)); - DocEntry::UP ret(new DocEntry(entry->getTimestamp(), - entry->getFlags(), - std::move(filtered), - entry->getPersistedDocumentSize())); + Document::UP filtered(FieldSet::createDocumentSubsetCopy(*entry->getDocument(), *it->_fieldSet)); + auto ret = DocEntry::create(entry->getTimestamp(), std::move(filtered), entry->getSize()); entries.push_back(std::move(ret)); } else { // Use entry as-is. - entries.push_back(DocEntry::UP(entry->clone())); + entries.push_back(cloneDocEntry(*entry)); ++fastPath; } } diff --git a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h index a7a784d0479..3b6fc9f1449 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummypersistence.h @@ -9,6 +9,7 @@ #pragma once #include <vespa/persistence/spi/abstractpersistenceprovider.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/document/base/globalid.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/vespalib/stllike/hash_map.h> diff --git a/persistence/src/vespa/persistence/spi/docentry.cpp b/persistence/src/vespa/persistence/spi/docentry.cpp index 8669bbf666b..f0329e8cc5e 100644 --- a/persistence/src/vespa/persistence/spi/docentry.cpp +++ b/persistence/src/vespa/persistence/spi/docentry.cpp @@ -4,90 +4,104 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/objects/nbostream.h> #include <sstream> -#include <cassert> namespace storage::spi { -DocEntry::DocEntry(Timestamp t, int metaFlags, DocumentUP doc) - : _timestamp(t), - _metaFlags(metaFlags), - _persistedDocumentSize(doc->serialize().size()), - _size(_persistedDocumentSize + sizeof(DocEntry)), - _documentId(), +namespace { + +class DocEntryWithId final : public DocEntry { +public: + DocEntryWithId(Timestamp t, DocumentMetaEnum metaEnum, const DocumentId &docId); + ~DocEntryWithId(); + vespalib::string toString() const override; + const DocumentId* getDocumentId() const override { return & _documentId; } + vespalib::stringref getDocumentType() const override { return _documentId.getDocType(); } + GlobalId getGid() const override { return _documentId.getGlobalId(); } +private: + DocumentId _documentId; +}; + +class DocEntryWithTypeAndGid final : public DocEntry { +public: + DocEntryWithTypeAndGid(Timestamp t, DocumentMetaEnum metaEnum, vespalib::stringref docType, GlobalId gid); + ~DocEntryWithTypeAndGid(); + vespalib::string toString() const override; + vespalib::stringref getDocumentType() const override { return _type; } + GlobalId getGid() const override { return _gid; } +private: + vespalib::string _type; + GlobalId _gid; +}; + +class DocEntryWithDoc final : public DocEntry { +public: + DocEntryWithDoc(Timestamp t, DocumentUP doc); + + /** + * Constructor that can be used by providers that already know + * the serialized size of the document, so the potentially expensive + * call to getSerializedSize can be avoided. This value shall be the size of the document _before_ + * any field filtering is performed. + */ + DocEntryWithDoc(Timestamp t, DocumentUP doc, size_t serializedDocumentSize); + ~DocEntryWithDoc(); + vespalib::string toString() const override; + const Document* getDocument() const override { return _document.get(); } + const DocumentId* getDocumentId() const override { return &_document->getId(); } + DocumentUP releaseDocument() override { return std::move(_document); } + vespalib::stringref getDocumentType() const override { return _document->getId().getDocType(); } + GlobalId getGid() const override { return _document->getId().getGlobalId(); } +private: + DocumentUP _document; +}; + +DocEntryWithDoc::DocEntryWithDoc(Timestamp t, DocumentUP doc) + : DocEntry(t, DocumentMetaEnum::NONE, doc->serialize().size()), _document(std::move(doc)) { } -DocEntry::DocEntry(Timestamp t, int metaFlags, DocumentUP doc, size_t serializedDocumentSize) - : _timestamp(t), - _metaFlags(metaFlags), - _persistedDocumentSize(serializedDocumentSize), - _size(_persistedDocumentSize + sizeof(DocEntry)), - _documentId(), +DocEntryWithDoc::DocEntryWithDoc(Timestamp t, DocumentUP doc, size_t serializedDocumentSize) + : DocEntry(t, DocumentMetaEnum::NONE, serializedDocumentSize), _document(std::move(doc)) { } -DocEntry::DocEntry(Timestamp t, int metaFlags, const DocumentId& docId) - : _timestamp(t), - _metaFlags(metaFlags), - _persistedDocumentSize(docId.getSerializedSize()), - _size(_persistedDocumentSize + sizeof(DocEntry)), - _documentId(new DocumentId(docId)), - _document() +DocEntryWithId::DocEntryWithId(Timestamp t, DocumentMetaEnum metaEnum, const DocumentId& docId) + : DocEntry(t, metaEnum, docId.getSerializedSize()), + _documentId(docId) { } -DocEntry::DocEntry(Timestamp t, int metaFlags) - : _timestamp(t), - _metaFlags(metaFlags), - _persistedDocumentSize(0), - _size(sizeof(DocEntry)), - _documentId(), - _document() +DocEntryWithTypeAndGid::DocEntryWithTypeAndGid(Timestamp t, DocumentMetaEnum metaEnum, vespalib::stringref docType, GlobalId gid) + : DocEntry(t, metaEnum, docType.size() + sizeof(gid)), + _type(docType), + _gid(gid) { } -DocEntry::~DocEntry() = default; - -DocEntry* -DocEntry::clone() const { - DocEntry* ret; - if (_documentId) { - ret = new DocEntry(_timestamp, _metaFlags, *_documentId); - ret->setPersistedDocumentSize(_persistedDocumentSize); - } else if (_document) { - ret = new DocEntry(_timestamp, _metaFlags, - std::make_unique<Document>(*_document), - _persistedDocumentSize); - } else { - ret = new DocEntry(_timestamp, _metaFlags); - ret->setPersistedDocumentSize(_persistedDocumentSize); - } - return ret; -} - -const DocumentId* -DocEntry::getDocumentId() const { - return (_document ? &_document->getId() : _documentId.get()); -} +DocEntryWithTypeAndGid::~DocEntryWithTypeAndGid() = default; +DocEntryWithId::~DocEntryWithId() = default; +DocEntryWithDoc::~DocEntryWithDoc() = default; -DocumentUP -DocEntry::releaseDocument() { - return std::move(_document); +vespalib::string +DocEntryWithId::toString() const +{ + std::ostringstream out; + out << "DocEntry(" << getTimestamp() << ", " << int(getMetaEnum()) << ", " << _documentId << ")"; + return out.str(); } -DocEntry::SizeType -DocEntry::getDocumentSize() const +vespalib::string +DocEntryWithTypeAndGid::toString() const { - assert(_size >= sizeof(DocEntry)); - return _size - sizeof(DocEntry); + std::ostringstream out; + out << "DocEntry(" << getTimestamp() << ", " << int(getMetaEnum()) << ", " << _type << ", " << _gid << ")"; + return out.str(); } vespalib::string -DocEntry::toString() const +DocEntryWithDoc::toString() const { std::ostringstream out; - out << "DocEntry(" << _timestamp << ", " << _metaFlags << ", "; - if (_documentId) { - out << *_documentId; - } else if (_document.get()) { + out << "DocEntry(" << getTimestamp() << ", " << int(getMetaEnum()) << ", "; + if (_document.get()) { out << "Doc(" << _document->getId() << ")"; } else { out << "metadata only"; @@ -96,53 +110,47 @@ DocEntry::toString() const return out.str(); } -std::ostream & -operator << (std::ostream & os, const DocEntry & r) { - return os << r.toString(); } -bool -DocEntry::operator==(const DocEntry& entry) const { - if (_timestamp != entry._timestamp) { - return false; - } - - if (_metaFlags != entry._metaFlags) { - return false; - } - - if (_documentId) { - if (!entry._documentId) { - return false; - } +DocEntry::UP +DocEntry::create(Timestamp t, DocumentMetaEnum metaEnum) { + return UP(new DocEntry(t, metaEnum)); +} +DocEntry::UP +DocEntry::create(Timestamp t, DocumentMetaEnum metaEnum, const DocumentId &docId) { + return std::make_unique<DocEntryWithId>(t, metaEnum, docId); +} +DocEntry::UP +DocEntry::create(Timestamp t, DocumentMetaEnum metaEnum, vespalib::stringref docType, GlobalId gid) { + return std::make_unique<DocEntryWithTypeAndGid>(t, metaEnum, docType, gid); +} +DocEntry::UP +DocEntry::create(Timestamp t, DocumentUP doc) { + return std::make_unique<DocEntryWithDoc>(t, std::move(doc)); +} +DocEntry::UP +DocEntry::create(Timestamp t, DocumentUP doc, SizeType serializedDocumentSize) { + return std::make_unique<DocEntryWithDoc>(t, std::move(doc), serializedDocumentSize); +} - if (*_documentId != *entry._documentId) { - return false; - } - } else { - if (entry._documentId) { - return false; - } - } +DocEntry::~DocEntry() = default; - if (_document) { - if (!entry._document) { - return false; - } +DocumentUP +DocEntry::releaseDocument() { + return {}; +} - if (*_document != *entry._document) { - return false; - } - } else { - if (entry._document) { - return false; - } - } - if (_persistedDocumentSize != entry._persistedDocumentSize) { - return false; - } +vespalib::string +DocEntry::toString() const +{ + std::ostringstream out; + out << "DocEntry(" << _timestamp << ", " << int(_metaEnum) << ", metadata only)"; + return out.str(); +} - return true; +std::ostream & +operator << (std::ostream & os, const DocEntry & r) { + return os << r.toString(); } } diff --git a/persistence/src/vespa/persistence/spi/docentry.h b/persistence/src/vespa/persistence/spi/docentry.h index 3374ef6c02d..9ad06b41e90 100644 --- a/persistence/src/vespa/persistence/spi/docentry.h +++ b/persistence/src/vespa/persistence/spi/docentry.h @@ -14,80 +14,59 @@ #pragma once #include <persistence/spi/types.h> +#include <vespa/document/base/globalid.h> namespace storage::spi { -enum DocumentMetaFlags { +enum class DocumentMetaEnum { NONE = 0x0, REMOVE_ENTRY = 0x1 }; class DocEntry { public: - typedef uint32_t SizeType; -private: - Timestamp _timestamp; - int _metaFlags; - SizeType _persistedDocumentSize; - SizeType _size; - DocumentIdUP _documentId; - DocumentUP _document; -public: + using SizeType = uint32_t; using UP = std::unique_ptr<DocEntry>; using SP = std::shared_ptr<DocEntry>; - DocEntry(Timestamp t, int metaFlags, DocumentUP doc); - - /** - * Constructor that can be used by providers that already know - * the serialized size of the document, so the potentially expensive - * call to getSerializedSize can be avoided. - */ - DocEntry(Timestamp t, int metaFlags, DocumentUP doc, size_t serializedDocumentSize); - DocEntry(Timestamp t, int metaFlags, const DocumentId& docId); - - DocEntry(Timestamp t, int metaFlags); - ~DocEntry(); - DocEntry* clone() const; - const Document* getDocument() const { return _document.get(); } - const DocumentId* getDocumentId() const; - DocumentUP releaseDocument(); - bool isRemove() const { return (_metaFlags & REMOVE_ENTRY); } + DocEntry(const DocEntry &) = delete; + DocEntry & operator=(const DocEntry &) = delete; + DocEntry(DocEntry &&) = delete; + DocEntry & operator=(DocEntry &&) = delete; + virtual ~DocEntry(); + bool isRemove() const { return (_metaEnum == DocumentMetaEnum::REMOVE_ENTRY); } Timestamp getTimestamp() const { return _timestamp; } - int getFlags() const { return _metaFlags; } - void setFlags(int flags) { _metaFlags = flags; } - /** - * @return In-memory size of this doc entry, including document instance. - * In essence: serialized size of document + sizeof(DocEntry). - */ - SizeType getSize() const { return _size; } + DocumentMetaEnum getMetaEnum() const { return _metaEnum; } /** * If entry contains a document, returns its serialized size. * If entry contains a document id, returns the serialized size of * the id alone. - * Otherwise (i.e. metadata only), returns zero. - */ - SizeType getDocumentSize() const; - /** - * Return size of document as it exists in persisted form. By default - * this will return the serialized size of the entry's document instance, - * but for persistence providers that are able to provide this information - * efficiently, this value can be set explicitly to provide better statistical - * tracking for e.g. visiting operations in the service layer. - * If explicitly set, this value shall be the size of the document _before_ - * any field filtering is performed. + * Otherwise (i.e. metadata only), returns sizeof(DocEntry). */ - SizeType getPersistedDocumentSize() const { return _persistedDocumentSize; } - /** - * Set persisted size of document. Optional. - * @see getPersistedDocumentSize - */ - void setPersistedDocumentSize(SizeType persistedDocumentSize) { - _persistedDocumentSize = persistedDocumentSize; - } + SizeType getSize() const { return _size; } - vespalib::string toString() const; - bool operator==(const DocEntry& entry) const; + virtual vespalib::string toString() const; + virtual const Document* getDocument() const { return nullptr; } + virtual const DocumentId* getDocumentId() const { return nullptr; } + virtual vespalib::stringref getDocumentType() const { return vespalib::stringref(); } + virtual GlobalId getGid() const { return GlobalId(); } + virtual DocumentUP releaseDocument(); + static UP create(Timestamp t, DocumentMetaEnum metaEnum); + static UP create(Timestamp t, DocumentMetaEnum metaEnum, const DocumentId &docId); + static UP create(Timestamp t, DocumentMetaEnum metaEnum, vespalib::stringref docType, GlobalId gid); + static UP create(Timestamp t, DocumentUP doc); + static UP create(Timestamp t, DocumentUP doc, SizeType serializedDocumentSize); +protected: + DocEntry(Timestamp t, DocumentMetaEnum metaEnum, SizeType size) + : _timestamp(t), + _metaEnum(metaEnum), + _size(size) + {} +private: + DocEntry(Timestamp t, DocumentMetaEnum metaEnum) : DocEntry(t, metaEnum, sizeof(DocEntry)) { } + Timestamp _timestamp; + DocumentMetaEnum _metaEnum; + SizeType _size; }; std::ostream & operator << (std::ostream & os, const DocEntry & r); diff --git a/persistence/src/vespa/persistence/spi/result.cpp b/persistence/src/vespa/persistence/spi/result.cpp index e458d58fe69..a728f93e60a 100644 --- a/persistence/src/vespa/persistence/spi/result.cpp +++ b/persistence/src/vespa/persistence/spi/result.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "result.h" +#include "docentry.h" #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/stllike/asciistream.h> #include <ostream> @@ -48,6 +49,24 @@ GetResult::~GetResult() = default; BucketIdListResult::~BucketIdListResult() = default; IterateResult::~IterateResult() = default; +IterateResult::IterateResult(IterateResult &&) noexcept = default; +IterateResult & IterateResult::operator=(IterateResult &&) noexcept = default; + +IterateResult::IterateResult(ErrorType error, const vespalib::string& errorMessage) + : Result(error, errorMessage), + _completed(false) +{ } + + +IterateResult::IterateResult(List entries, bool completed) + : _completed(completed), + _entries(std::move(entries)) +{ } + +IterateResult::List +IterateResult::steal_entries() { + return std::move(_entries); +} } diff --git a/persistence/src/vespa/persistence/spi/result.h b/persistence/src/vespa/persistence/spi/result.h index c734a885b12..10c589307ba 100644 --- a/persistence/src/vespa/persistence/spi/result.h +++ b/persistence/src/vespa/persistence/spi/result.h @@ -3,11 +3,12 @@ #include "bucketinfo.h" #include "bucket.h" -#include "docentry.h" #include <vespa/document/bucket/bucketidlist.h> namespace storage::spi { +class DocEntry; + class Result { public: typedef std::unique_ptr<Result> UP; @@ -279,15 +280,12 @@ private: class IterateResult : public Result { public: - typedef std::vector<DocEntry::UP> List; + using List = std::vector<std::unique_ptr<DocEntry>>; /** * Constructor used when there was an error creating the iterator. */ - IterateResult(ErrorType error, const vespalib::string& errorMessage) - : Result(error, errorMessage), - _completed(false) - { } + IterateResult(ErrorType error, const vespalib::string& errorMessage); /** * Constructor used when the iteration was successful. @@ -296,24 +294,21 @@ public: * * @param completed Set to true if iteration has been completed. */ - IterateResult(List entries, bool completed) - : _completed(completed), - _entries(std::move(entries)) - { } + IterateResult(List entries, bool completed); IterateResult(const IterateResult &) = delete; - IterateResult(IterateResult &&rhs) noexcept = default; - IterateResult &operator=(IterateResult &&rhs) noexcept = default; + IterateResult(IterateResult &&rhs) noexcept; + IterateResult &operator=(IterateResult &&rhs) noexcept; ~IterateResult(); const List& getEntries() const { return _entries; } - List steal_entries() { return std::move(_entries); } + List steal_entries(); bool isCompleted() const { return _completed; } private: bool _completed; - std::vector<DocEntry::UP> _entries; + List _entries; }; } diff --git a/persistence/src/vespa/persistence/spi/test.cpp b/persistence/src/vespa/persistence/spi/test.cpp index 32381110630..58a8ce3fe52 100644 --- a/persistence/src/vespa/persistence/spi/test.cpp +++ b/persistence/src/vespa/persistence/spi/test.cpp @@ -1,7 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "test.h" +#include "docentry.h" #include <vespa/document/test/make_bucket_space.h> +#include <vespa/document/fieldvalue/document.h> using document::BucketId; using document::BucketSpace; @@ -9,9 +11,45 @@ using document::test::makeBucketSpace; namespace storage::spi::test { -Bucket makeSpiBucket(BucketId bucketId) +Bucket +makeSpiBucket(BucketId bucketId) { return Bucket(document::Bucket(makeBucketSpace(), bucketId)); } +std::unique_ptr<DocEntry> +cloneDocEntry(const DocEntry & e) { + std::unique_ptr<DocEntry> ret; + if (e.getDocument()) { + ret = DocEntry::create(e.getTimestamp(), std::make_unique<Document>(*e.getDocument()), e.getSize()); + } else if (e.getDocumentId()) { + ret = DocEntry::create(e.getTimestamp(), e.getMetaEnum(), *e.getDocumentId()); + } else { + ret = DocEntry::create(e.getTimestamp(), e.getMetaEnum()); + } + return ret; +} + +bool +equal(const DocEntry & a, const DocEntry & b) { + if (a.getTimestamp() != b.getTimestamp()) return false; + if (a.getMetaEnum() != b.getMetaEnum()) return false; + if (a.getSize() != b.getSize()) return false; + + if (a.getDocument()) { + if (!b.getDocument()) return false; + if (*a.getDocument() != *b.getDocument()) return false; + } else { + if (b.getDocument()) return false; + } + if (a.getDocumentId()) { + if (!b.getDocumentId()) return false; + if (*a.getDocumentId() != *b.getDocumentId()) return false; + } else { + if (b.getDocumentId()) return false; + } + + return true; +} + } diff --git a/persistence/src/vespa/persistence/spi/test.h b/persistence/src/vespa/persistence/spi/test.h index af7109ec80c..1660e5f14fd 100644 --- a/persistence/src/vespa/persistence/spi/test.h +++ b/persistence/src/vespa/persistence/spi/test.h @@ -3,11 +3,16 @@ #pragma once #include "bucket.h" +#include <memory> + +namespace storage::spi { class DocEntry; } namespace storage::spi::test { // Helper functions used by unit tests Bucket makeSpiBucket(document::BucketId bucketId); +std::unique_ptr<DocEntry> cloneDocEntry(const DocEntry & entry); +bool equal(const DocEntry & a, const DocEntry & b); } diff --git a/screwdriver/build-vespa.sh b/screwdriver/build-vespa.sh index f321e3820fd..886144b1c12 100755 --- a/screwdriver/build-vespa.sh +++ b/screwdriver/build-vespa.sh @@ -6,7 +6,7 @@ set -e readonly SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd )" readonly NUM_THREADS=$(( $(nproc) + 2 )) -source /etc/profile.d/enable-devtoolset-10.sh +source /etc/profile.d/enable-devtoolset-11.sh source /etc/profile.d/enable-rh-maven35.sh source /etc/profile.d/enable-rh-git227.sh @@ -57,9 +57,9 @@ esac if [[ $SHOULD_BUILD == systemtest ]]; then yum -y --setopt=skip_missing_names_on_install=False install \ zstd \ - devtoolset-10-gcc-c++ \ - devtoolset-10-libatomic-devel \ - devtoolset-10-binutils \ + devtoolset-11-gcc-c++ \ + devtoolset-11-libatomic-devel \ + devtoolset-11-binutils \ libxml2-devel \ rh-ruby27-rubygems-devel \ rh-ruby27-ruby-devel \ diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index c63ccfe4e24..7fb98cb2514 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -61,8 +61,8 @@ vespa_define_module( src/tests/proton/attribute/attribute_initializer src/tests/proton/attribute/attribute_manager src/tests/proton/attribute/attribute_populator + src/tests/proton/attribute/attribute_transient_memory_calculator src/tests/proton/attribute/attribute_usage_filter - src/tests/proton/attribute/attribute_usage_sampler_functor src/tests/proton/attribute/attribute_usage_stats src/tests/proton/attribute/attributes_state_explorer src/tests/proton/attribute/document_field_extractor diff --git a/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/CMakeLists.txt new file mode 100644 index 00000000000..bc77f2d949b --- /dev/null +++ b/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_attribute_transient_memory_calculator_test_app TEST + SOURCES + attribute_transient_memory_calculator_test.cpp + DEPENDS + searchcore_attribute + searchcore_pcommon + GTest::GTest +) +vespa_add_test(NAME searchcore_attribute_transient_memory_calculator_test_app COMMAND searchcore_attribute_transient_memory_calculator_test_app) diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/attribute_usage_sampler_functor_test.cpp b/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/attribute_transient_memory_calculator_test.cpp index 31f90075421..786d1f6a801 100644 --- a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/attribute_usage_sampler_functor_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_transient_memory_calculator/attribute_transient_memory_calculator_test.cpp @@ -5,10 +5,7 @@ #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/integerbase.h> #include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> -#include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> -#include <vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h> -#include <vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h> -#include <vespa/searchcore/proton/common/transient_resource_usage_provider.h> +#include <vespa/searchcore/proton/attribute/attribute_transient_memory_calculator.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/string.h> @@ -58,73 +55,32 @@ std::shared_ptr<AttributeVector> build_attribute_vector(const vespalib::string& } -class AttributeUsageSamplerFunctorTest : public ::testing::Test { -protected: - AttributeUsageFilter _filter; - std::shared_ptr<TransientResourceUsageProvider> _transient_usage_provider; -public: - AttributeUsageSamplerFunctorTest(); - ~AttributeUsageSamplerFunctorTest(); -protected: - void sample_usage(bool sample_a1, bool sample_a2, bool old_fast_search, bool new_fast_search = true); - size_t get_transient_memory_usage() const { return _transient_usage_provider->get_transient_memory_usage(); } -}; - -AttributeUsageSamplerFunctorTest::AttributeUsageSamplerFunctorTest() - : _filter(), - _transient_usage_provider(std::make_shared<TransientResourceUsageProvider>()) -{ -} - -AttributeUsageSamplerFunctorTest::~AttributeUsageSamplerFunctorTest() = default; - -void -AttributeUsageSamplerFunctorTest::sample_usage(bool sample_a1, bool sample_a2, bool old_fast_search, bool new_fast_search) +size_t +sample_usage(bool old_fast_search, bool new_fast_search) { auto old_config = build_config(old_fast_search); auto old_inspector = std::make_shared<AttributeConfigInspector>(old_config); auto av1 = build_attribute_vector("a1", *old_inspector, 1); - auto av2 = build_attribute_vector("a2", *old_inspector, 3); EXPECT_EQ(av1->getEnumeratedSave(), old_fast_search); auto new_config = build_config(new_fast_search); auto new_inspector = std::make_shared<AttributeConfigInspector>(new_config); - auto context = std::make_shared<AttributeUsageSamplerContext>(_filter, new_inspector, _transient_usage_provider); - if (sample_a1) { - AttributeUsageSamplerFunctor functor1(context, "ready"); - functor1(*av1); - } - if (sample_a2) { - AttributeUsageSamplerFunctor functor2(context, "ready"); - functor2(*av2); - } -} - -TEST_F(AttributeUsageSamplerFunctorTest, plain_attribute_vector_requires_no_transient_memory_for_load) -{ - sample_usage(true, true, false, false); - EXPECT_EQ(0u, get_transient_memory_usage()); + AttributeTransientMemoryCalculator calc; + return calc(*av1, *new_inspector->get_config("a1")); } -TEST_F(AttributeUsageSamplerFunctorTest, fast_search_attribute_vector_requires_transient_memory_for_load) +TEST(AttributeTransientMemoryCalculator, plain_attribute_vector_requires_no_transient_memory_for_load) { - sample_usage(true, false, true, true); - EXPECT_EQ(24u, get_transient_memory_usage()); + EXPECT_EQ(0, sample_usage(false, false)); } -TEST_F(AttributeUsageSamplerFunctorTest, fast_search_attribute_vector_requires_more_transient_memory_for_load_from_unenumerated) +TEST(AttributeTransientMemoryCalculator, fast_search_attribute_vector_requires_transient_memory_for_load) { - sample_usage(true, false, false, true); - EXPECT_EQ(40u, get_transient_memory_usage()); + EXPECT_EQ(24u, sample_usage(true, true)); } -TEST_F(AttributeUsageSamplerFunctorTest, transient_memory_aggregation_function_for_attribute_usage_sampler_context_is_max) +TEST(AttributeTransientMemoryCalculator, fast_search_attribute_vector_requires_more_transient_memory_for_load_from_unenumerated) { - sample_usage(true, false, true, true); - EXPECT_EQ(24u, get_transient_memory_usage()); - sample_usage(false, true, true, true); - EXPECT_EQ(72u, get_transient_memory_usage()); - sample_usage(true, true, true, true); - EXPECT_EQ(72u, get_transient_memory_usage()); + EXPECT_EQ(40u, sample_usage(false, true)); } } diff --git a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt b/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt deleted file mode 100644 index 6034f5a1309..00000000000 --- a/searchcore/src/tests/proton/attribute/attribute_usage_sampler_functor/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_attribute_usage_sampler_functor_test_app TEST - SOURCES - attribute_usage_sampler_functor_test.cpp - DEPENDS - searchcore_attribute - searchcore_pcommon - GTest::GTest -) -vespa_add_test(NAME searchcore_attribute_usage_sampler_functor_test_app COMMAND searchcore_attribute_usage_sampler_functor_test_app) diff --git a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp index fab46e61494..82eac88a53e 100644 --- a/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp +++ b/searchcore/src/tests/proton/document_iterator/document_iterator_test.cpp @@ -48,7 +48,9 @@ using storage::spi::IncludedVersions; using storage::spi::IterateResult; using storage::spi::Selection; using storage::spi::Timestamp; +using storage::spi::DocumentMetaEnum; using storage::spi::test::makeSpiBucket; +using storage::spi::test::equal; using namespace proton; @@ -274,18 +276,14 @@ struct PairDR : DocumentRetrieverBaseForTest { } }; -size_t getSize() { - return sizeof(DocEntry); -} - size_t getSize(const document::Document &doc) { vespalib::nbostream tmp; doc.serialize(tmp); - return tmp.size() + getSize(); + return tmp.size(); } size_t getSize(const document::DocumentId &id) { - return id.getSerializedSize() + getSize(); + return id.getSerializedSize(); } IDocumentRetriever::SP nil() { return std::make_unique<UnitDR>(); } @@ -386,19 +384,19 @@ void checkDoc(const IDocumentRetriever &dr, const std::string &id, EXPECT_TRUE(DocumentId(id) == doc->getId()); } -void checkEntry(const IterateResult &res, size_t idx, const Timestamp ×tamp, int flags) +void checkEntry(const IterateResult &res, size_t idx, const Timestamp ×tamp, DocumentMetaEnum flags) { ASSERT_LESS(idx, res.getEntries().size()); - DocEntry expect(timestamp, flags); - EXPECT_EQUAL(expect, *res.getEntries()[idx]); - EXPECT_EQUAL(getSize(), res.getEntries()[idx]->getSize()); + auto expect = DocEntry::create(timestamp, flags); + EXPECT_TRUE(equal(*expect, *res.getEntries()[idx])); + EXPECT_EQUAL(sizeof(DocEntry), res.getEntries()[idx]->getSize()); } void checkEntry(const IterateResult &res, size_t idx, const DocumentId &id, const Timestamp ×tamp) { ASSERT_LESS(idx, res.getEntries().size()); - DocEntry expect(timestamp, storage::spi::REMOVE_ENTRY, id); - EXPECT_EQUAL(expect, *res.getEntries()[idx]); + auto expect = DocEntry::create(timestamp, DocumentMetaEnum::REMOVE_ENTRY, id); + EXPECT_TRUE(equal(*expect, *res.getEntries()[idx])); EXPECT_EQUAL(getSize(id), res.getEntries()[idx]->getSize()); EXPECT_GREATER(getSize(id), 0u); } @@ -406,8 +404,8 @@ void checkEntry(const IterateResult &res, size_t idx, const DocumentId &id, cons void checkEntry(const IterateResult &res, size_t idx, const Document &doc, const Timestamp ×tamp) { ASSERT_LESS(idx, res.getEntries().size()); - DocEntry expect(timestamp, storage::spi::NONE, Document::UP(doc.clone())); - EXPECT_EQUAL(expect, *res.getEntries()[idx]); + auto expect = DocEntry::create(timestamp, Document::UP(doc.clone())); + EXPECT_TRUE(equal(*expect, *res.getEntries()[idx])); EXPECT_EQUAL(getSize(doc), res.getEntries()[idx]->getSize()); EXPECT_GREATER(getSize(doc), 0u); } @@ -608,9 +606,9 @@ TEST("require that using an empty field set returns meta-data only") { IterateResult res = itr.iterate(largeNum); EXPECT_TRUE(res.isCompleted()); EXPECT_EQUAL(3u, res.getEntries().size()); - TEST_DO(checkEntry(res, 0, Timestamp(2), storage::spi::NONE)); - TEST_DO(checkEntry(res, 1, Timestamp(3), storage::spi::NONE)); - TEST_DO(checkEntry(res, 2, Timestamp(4), storage::spi::REMOVE_ENTRY)); + TEST_DO(checkEntry(res, 0, Timestamp(2), DocumentMetaEnum::NONE)); + TEST_DO(checkEntry(res, 1, Timestamp(3), DocumentMetaEnum::NONE)); + TEST_DO(checkEntry(res, 2, Timestamp(4), DocumentMetaEnum::REMOVE_ENTRY)); } TEST("require that entries in other buckets are skipped") { @@ -650,15 +648,15 @@ TEST("require that maxBytes splits iteration results for meta-data only iteratio itr.add(doc("id:ns:document::1", Timestamp(2), bucket(5))); itr.add(cat(rem("id:ns:document::2", Timestamp(3), bucket(5)), doc("id:ns:document::3", Timestamp(4), bucket(5)))); - IterateResult res1 = itr.iterate(getSize() + getSize()); + IterateResult res1 = itr.iterate(2 * sizeof(DocEntry)); EXPECT_TRUE(!res1.isCompleted()); EXPECT_EQUAL(2u, res1.getEntries().size()); - TEST_DO(checkEntry(res1, 0, Timestamp(2), storage::spi::NONE)); - TEST_DO(checkEntry(res1, 1, Timestamp(3), storage::spi::REMOVE_ENTRY)); + TEST_DO(checkEntry(res1, 0, Timestamp(2), DocumentMetaEnum::NONE)); + TEST_DO(checkEntry(res1, 1, Timestamp(3), DocumentMetaEnum::REMOVE_ENTRY)); IterateResult res2 = itr.iterate(largeNum); EXPECT_TRUE(res2.isCompleted()); - TEST_DO(checkEntry(res2, 0, Timestamp(4), storage::spi::NONE)); + TEST_DO(checkEntry(res2, 0, Timestamp(4), DocumentMetaEnum::NONE)); IterateResult res3 = itr.iterate(largeNum); EXPECT_TRUE(res3.isCompleted()); diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 227e885564d..9657619be40 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -1,14 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/config-attributes.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/test/make_bucket_space.h> +#include <vespa/fastos/thread.h> +#include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> #include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> #include <vespa/searchcore/proton/attribute/i_attribute_manager.h> #include <vespa/searchcore/proton/bucketdb/bucket_create_notifier.h> +#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/searchcore/proton/common/doctypename.h> -#include <vespa/searchcore/proton/common/transient_resource_usage_provider.h> -#include <vespa/searchcore/proton/documentmetastore/operation_listener.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> +#include <vespa/searchcore/proton/documentmetastore/operation_listener.h> #include <vespa/searchcore/proton/feedoperation/moveoperation.h> #include <vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h> #include <vespa/searchcore/proton/feedoperation/putoperation.h> @@ -25,14 +30,9 @@ #include <vespa/searchcore/proton/server/maintenancecontroller.h> #include <vespa/searchcore/proton/test/buckethandler.h> #include <vespa/searchcore/proton/test/clusterstatehandler.h> -#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> #include <vespa/searchcore/proton/test/mock_attribute_manager.h> #include <vespa/searchcore/proton/test/test.h> -#include <vespa/config-attributes.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/document/test/make_bucket_space.h> -#include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> #include <vespa/searchlib/common/idocumentmetastore.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/vespalib/data/slime/slime.h> @@ -43,7 +43,6 @@ #include <vespa/vespalib/util/monitored_refcount.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/fastos/thread.h> #include <unistd.h> #include <thread> @@ -824,8 +823,6 @@ MaintenanceControllerFixture::injectMaintenanceJobs() _bucketCreateNotifier, makeBucketSpace(), _fh, _fh, _bmc, _clusterStateHandler, _bucketHandler, _calc, _diskMemUsageNotifier, _jobTrackers, _readyAttributeManager, _notReadyAttributeManager, - std::make_unique<const AttributeConfigInspector>(AttributesConfigBuilder()), - std::make_shared<TransientResourceUsageProvider>(), _attributeUsageFilter); } } diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp index 855b31310a3..aff368ceced 100644 --- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp +++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp @@ -197,16 +197,16 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() { bool fret1 = DocumentSummary::readDocIdLimit(index_dir, fusionDocIdLimit); ASSERT_TRUE(fret1); SelectorArray selector(fusionDocIdLimit, 0); - bool fret2 = Fusion::merge(schema, - index_dir2, - fusionInputs, - selector, - false /* dynamicKPosOccFormat */, - tuneFileIndexing, - fileHeaderContext, - sharedExecutor, - std::make_shared<FlushToken>()); - ASSERT_TRUE(fret2); + { + Fusion fusion(schema, + index_dir2, + fusionInputs, + selector, + tuneFileIndexing, + fileHeaderContext); + bool fret2 = fusion.merge(sharedExecutor, std::make_shared<FlushToken>()); + ASSERT_TRUE(fret2); + } // Fusion test with all docs removed in output (doesn't affect word list) const string index_dir3 = "test_index3"; @@ -216,16 +216,16 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() { bool fret3 = DocumentSummary::readDocIdLimit(index_dir, fusionDocIdLimit); ASSERT_TRUE(fret3); SelectorArray selector2(fusionDocIdLimit, 1); - bool fret4 = Fusion::merge(schema, - index_dir3, - fusionInputs, - selector2, - false /* dynamicKPosOccFormat */, - tuneFileIndexing, - fileHeaderContext, - sharedExecutor, - std::make_shared<FlushToken>()); - ASSERT_TRUE(fret4); + { + Fusion fusion(schema, + index_dir3, + fusionInputs, + selector2, + tuneFileIndexing, + fileHeaderContext); + bool fret4 = fusion.merge(sharedExecutor, std::make_shared<FlushToken>()); + ASSERT_TRUE(fret4); + } // Fusion test with all docs removed in input (affects word list) const string index_dir4 = "test_index4"; @@ -235,16 +235,17 @@ void Test::requireThatMemoryIndexCanBeDumpedAndSearched() { bool fret5 = DocumentSummary::readDocIdLimit(index_dir3, fusionDocIdLimit); ASSERT_TRUE(fret5); SelectorArray selector3(fusionDocIdLimit, 0); - bool fret6 = Fusion::merge(schema, - index_dir4, - fusionInputs, - selector3, - false /* dynamicKPosOccFormat */, - tuneFileIndexing, - fileHeaderContext, - sharedExecutor, - std::make_shared<FlushToken>()); - ASSERT_TRUE(fret6); + { + Fusion fusion(schema, + index_dir4, + fusionInputs, + selector3, + tuneFileIndexing, + fileHeaderContext); + bool fret6 = fusion.merge(sharedExecutor, + std::make_shared<FlushToken>()); + ASSERT_TRUE(fret6); + } DiskIndex disk_index(index_dir); ASSERT_TRUE(disk_index.setup(TuneFileSearch())); diff --git a/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp b/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp index 6eab2c5bf3d..c814bdf3f37 100644 --- a/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp +++ b/searchcore/src/tests/proton/index/diskindexcleaner_test.cpp @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Unit tests for diskindexcleaner. -#include <vespa/searchcorespi/index/activediskindexes.h> +#include <vespa/searchcorespi/index/disk_indexes.h> #include <vespa/searchcorespi/index/diskindexcleaner.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/fastos/file.h> @@ -90,8 +90,8 @@ void createIndexes() { void Test::requireThatAllIndexesOlderThanLastFusionIsRemoved() { createIndexes(); - ActiveDiskIndexes active_indexes; - DiskIndexCleaner::clean(index_dir, active_indexes); + DiskIndexes disk_indexes; + DiskIndexCleaner::clean(index_dir, disk_indexes); vector<string> indexes = readIndexes(); EXPECT_EQUAL(3u, indexes.size()); EXPECT_TRUE(contains(indexes, "index.fusion.2")); @@ -101,17 +101,17 @@ void Test::requireThatAllIndexesOlderThanLastFusionIsRemoved() { void Test::requireThatIndexesInUseAreNotRemoved() { createIndexes(); - ActiveDiskIndexes active_indexes; - active_indexes.setActive(index_dir + "/index.fusion.1"); - active_indexes.setActive(index_dir + "/index.flush.2"); - DiskIndexCleaner::clean(index_dir, active_indexes); + DiskIndexes disk_indexes; + disk_indexes.setActive(index_dir + "/index.fusion.1", 0); + disk_indexes.setActive(index_dir + "/index.flush.2", 0); + DiskIndexCleaner::clean(index_dir, disk_indexes); vector<string> indexes = readIndexes(); EXPECT_TRUE(contains(indexes, "index.fusion.1")); EXPECT_TRUE(contains(indexes, "index.flush.2")); - active_indexes.notActive(index_dir + "/index.fusion.1"); - active_indexes.notActive(index_dir + "/index.flush.2"); - DiskIndexCleaner::clean(index_dir, active_indexes); + disk_indexes.notActive(index_dir + "/index.fusion.1"); + disk_indexes.notActive(index_dir + "/index.flush.2"); + DiskIndexCleaner::clean(index_dir, disk_indexes); indexes = readIndexes(); EXPECT_TRUE(!contains(indexes, "index.fusion.1")); EXPECT_TRUE(!contains(indexes, "index.flush.2")); @@ -120,8 +120,8 @@ void Test::requireThatIndexesInUseAreNotRemoved() { void Test::requireThatInvalidFlushIndexesAreRemoved() { createIndexes(); FastOS_File((index_dir + "/index.flush.4/serial.dat").c_str()).Delete(); - ActiveDiskIndexes active_indexes; - DiskIndexCleaner::clean(index_dir, active_indexes); + DiskIndexes disk_indexes; + DiskIndexCleaner::clean(index_dir, disk_indexes); vector<string> indexes = readIndexes(); EXPECT_EQUAL(2u, indexes.size()); EXPECT_TRUE(contains(indexes, "index.fusion.2")); @@ -131,8 +131,8 @@ void Test::requireThatInvalidFlushIndexesAreRemoved() { void Test::requireThatInvalidFusionIndexesAreRemoved() { createIndexes(); FastOS_File((index_dir + "/index.fusion.2/serial.dat").c_str()).Delete(); - ActiveDiskIndexes active_indexes; - DiskIndexCleaner::clean(index_dir, active_indexes); + DiskIndexes disk_indexes; + DiskIndexCleaner::clean(index_dir, disk_indexes); vector<string> indexes = readIndexes(); EXPECT_EQUAL(4u, indexes.size()); EXPECT_TRUE(contains(indexes, "index.fusion.1")); @@ -144,8 +144,8 @@ void Test::requireThatInvalidFusionIndexesAreRemoved() { void Test::requireThatRemoveDontTouchNewIndexes() { createIndexes(); FastOS_File((index_dir + "/index.flush.4/serial.dat").c_str()).Delete(); - ActiveDiskIndexes active_indexes; - DiskIndexCleaner::removeOldIndexes(index_dir, active_indexes); + DiskIndexes disk_indexes; + DiskIndexCleaner::removeOldIndexes(index_dir, disk_indexes); vector<string> indexes = readIndexes(); EXPECT_EQUAL(3u, indexes.size()); EXPECT_TRUE(contains(indexes, "index.fusion.2")); diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/CMakeLists.txt b/searchcore/src/tests/proton/server/disk_mem_usage_filter/CMakeLists.txt index 0bd8ca368e0..942ac6a8c69 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_filter/CMakeLists.txt +++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchcore_disk_mem_usage_filter_test_app TEST disk_mem_usage_filter_test.cpp DEPENDS searchcore_server + GTest::GTest ) vespa_add_test(NAME searchcore_disk_mem_usage_filter_test_app COMMAND searchcore_disk_mem_usage_filter_test_app) diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp index 4fbe7a13acb..ce85517ee09 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp @@ -1,21 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/server/disk_mem_usage_filter.h> #include <vespa/searchcore/proton/server/resource_usage_state.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace proton; namespace fs = std::filesystem; -struct Fixture +struct DiskMemUsageFilterTest : public ::testing::Test { DiskMemUsageFilter _filter; using State = DiskMemUsageFilter::State; using Config = DiskMemUsageFilter::Config; - Fixture() + DiskMemUsageFilterTest() : _filter(HwInfo(HwInfo::Disk(100, false, false), HwInfo::Memory(1000), HwInfo::Cpu(0))) { _filter.setDiskUsedSize(20); @@ -31,12 +31,12 @@ struct Fixture EXPECT_TRUE(_filter.acceptWriteOperation()); State state = _filter.getAcceptState(); EXPECT_TRUE(state.acceptWriteOperation()); - EXPECT_EQUAL(exp, state.message()); + EXPECT_EQ(exp, state.message()); } else { EXPECT_FALSE(_filter.acceptWriteOperation()); State state = _filter.getAcceptState(); EXPECT_FALSE(state.acceptWriteOperation()); - EXPECT_EQUAL(exp, state.message()); + EXPECT_EQ(exp, state.message()); } } @@ -54,71 +54,85 @@ struct Fixture } }; -TEST_F("Check that default filter allows write", Fixture) +TEST_F(DiskMemUsageFilterTest, default_filter_allows_write) { - f.testWrite(""); + testWrite(""); } -TEST_F("Check that stats are wired through", Fixture) +TEST_F(DiskMemUsageFilterTest, stats_are_wired_through) { - EXPECT_EQUAL(42u, f._filter.getMemoryStats().getMappingsCount()); - f.triggerMemoryLimit(); - EXPECT_EQUAL(43u, f._filter.getMemoryStats().getMappingsCount()); + EXPECT_EQ(42u, _filter.getMemoryStats().getMappingsCount()); + triggerMemoryLimit(); + EXPECT_EQ(43u, _filter.getMemoryStats().getMappingsCount()); } void assertResourceUsage(double usage, double limit, double utilization, const ResourceUsageState &state) { - EXPECT_EQUAL(usage, state.usage()); - EXPECT_EQUAL(limit, state.limit()); - EXPECT_EQUAL(utilization, state.utilization()); + EXPECT_EQ(usage, state.usage()); + EXPECT_EQ(limit, state.limit()); + EXPECT_DOUBLE_EQ(utilization, state.utilization()); } -TEST_F("Check that disk limit can be reached", Fixture) +TEST_F(DiskMemUsageFilterTest, disk_limit_can_be_reached) { - f._filter.setConfig(Fixture::Config(1.0, 0.8)); - TEST_DO(assertResourceUsage(0.2, 0.8, 0.25, f._filter.usageState().diskState())); - f.triggerDiskLimit(); - f.testWrite("diskLimitReached: { " - "action: \"add more content nodes\", " - "reason: \"disk used (0.9) > disk limit (0.8)\", " - "stats: { " - "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}"); - TEST_DO(assertResourceUsage(0.9, 0.8, 1.125, f._filter.usageState().diskState())); + _filter.setConfig(Config(1.0, 0.8)); + assertResourceUsage(0.2, 0.8, 0.25, _filter.usageState().diskState()); + triggerDiskLimit(); + testWrite("diskLimitReached: { " + "action: \"add more content nodes\", " + "reason: \"disk used (0.9) > disk limit (0.8)\", " + "stats: { " + "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}"); + assertResourceUsage(0.9, 0.8, 1.125, _filter.usageState().diskState()); } -TEST_F("Check that memory limit can be reached", Fixture) +TEST_F(DiskMemUsageFilterTest, memory_limit_can_be_reached) { - f._filter.setConfig(Fixture::Config(0.8, 1.0)); - TEST_DO(assertResourceUsage(0.3, 0.8, 0.375, f._filter.usageState().memoryState())); - f.triggerMemoryLimit(); - f.testWrite("memoryLimitReached: { " - "action: \"add more content nodes\", " - "reason: \"memory used (0.9) > memory limit (0.8)\", " - "stats: { " - "mapped: { virt: 897, rss: 898}, " - "anonymous: { virt: 899, rss: 900}, " - "physicalMemory: 1000, memoryUsed: 0.9, memoryLimit: 0.8}}"); - TEST_DO(assertResourceUsage(0.9, 0.8, 1.125, f._filter.usageState().memoryState())); + _filter.setConfig(Config(0.8, 1.0)); + assertResourceUsage(0.3, 0.8, 0.375, _filter.usageState().memoryState()); + triggerMemoryLimit(); + testWrite("memoryLimitReached: { " + "action: \"add more content nodes\", " + "reason: \"memory used (0.9) > memory limit (0.8)\", " + "stats: { " + "mapped: { virt: 897, rss: 898}, " + "anonymous: { virt: 899, rss: 900}, " + "physicalMemory: 1000, memoryUsed: 0.9, memoryLimit: 0.8}}"); + assertResourceUsage(0.9, 0.8, 1.125, _filter.usageState().memoryState()); } -TEST_F("Check that both disk limit and memory limit can be reached", Fixture) +TEST_F(DiskMemUsageFilterTest, both_disk_limit_and_memory_limit_can_be_reached) { - f._filter.setConfig(Fixture::Config(0.8, 0.8)); - f.triggerMemoryLimit(); - f.triggerDiskLimit(); - f.testWrite("memoryLimitReached: { " - "action: \"add more content nodes\", " - "reason: \"memory used (0.9) > memory limit (0.8)\", " - "stats: { " - "mapped: { virt: 897, rss: 898}, " - "anonymous: { virt: 899, rss: 900}, " - "physicalMemory: 1000, memoryUsed: 0.9, memoryLimit: 0.8}}, " - "diskLimitReached: { " - "action: \"add more content nodes\", " - "reason: \"disk used (0.9) > disk limit (0.8)\", " - "stats: { " - "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}"); + _filter.setConfig(Config(0.8, 0.8)); + triggerMemoryLimit(); + triggerDiskLimit(); + testWrite("memoryLimitReached: { " + "action: \"add more content nodes\", " + "reason: \"memory used (0.9) > memory limit (0.8)\", " + "stats: { " + "mapped: { virt: 897, rss: 898}, " + "anonymous: { virt: 899, rss: 900}, " + "physicalMemory: 1000, memoryUsed: 0.9, memoryLimit: 0.8}}, " + "diskLimitReached: { " + "action: \"add more content nodes\", " + "reason: \"disk used (0.9) > disk limit (0.8)\", " + "stats: { " + "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}"); } -TEST_MAIN() { TEST_RUN_ALL(); } +TEST_F(DiskMemUsageFilterTest, transient_disk_usage_is_tracked_in_usage_state_and_metrics) +{ + _filter.set_transient_resource_usage({40, 0}); + EXPECT_EQ(0.4, _filter.usageState().transient_disk_usage()); + EXPECT_EQ(0.4, _filter.get_metrics().get_transient_disk_usage()); +} + +TEST_F(DiskMemUsageFilterTest, transient_memory_usage_is_tracked_in_usage_state_and_metrics) +{ + _filter.set_transient_resource_usage({0, 200}); + EXPECT_EQ(0.2, _filter.usageState().transient_memory_usage()); + EXPECT_EQ(0.2, _filter.get_metrics().get_transient_memory_usage()); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp index 2731f51a85e..0c80553e1e7 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp +++ b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp @@ -34,8 +34,7 @@ public: : _memory_usage(memory_usage), _disk_usage(disk_usage) {} - size_t get_transient_memory_usage() const override { return _memory_usage; } - size_t get_transient_disk_usage() const override { return _disk_usage; } + TransientResourceUsage get_transient_resource_usage() const override { return {_disk_usage, _memory_usage}; } }; struct DiskMemUsageSamplerTest : public ::testing::Test { @@ -45,8 +44,8 @@ struct DiskMemUsageSamplerTest : public ::testing::Test { DiskMemUsageSampler::Config(0.8, 0.8, 50ms, make_hw_info())) { - sampler.add_transient_usage_provider(std::make_shared<MyProvider>(99, 200)); - sampler.add_transient_usage_provider(std::make_shared<MyProvider>(100, 199)); + sampler.add_transient_usage_provider(std::make_shared<MyProvider>(50, 200)); + sampler.add_transient_usage_provider(std::make_shared<MyProvider>(100, 150)); } const DiskMemUsageFilter& filter() const { return sampler.writeFilter(); } }; @@ -56,7 +55,7 @@ TEST_F(DiskMemUsageSamplerTest, resource_usage_is_sampled) // Poll for up to 20 seconds to get a sample. size_t i = 0; for (; i < static_cast<size_t>(20s / 50ms); ++i) { - if (filter().get_transient_memory_usage() > 0) { + if (filter().get_transient_resource_usage().memory() > 0) { break; } std::this_thread::sleep_for(50ms); @@ -70,10 +69,10 @@ TEST_F(DiskMemUsageSamplerTest, resource_usage_is_sampled) EXPECT_EQ(filter().getMemoryStats().getAnonymousRss(), 0); #endif EXPECT_GT(filter().getDiskUsedSize(), 0); - EXPECT_EQ(100, filter().get_transient_memory_usage()); - EXPECT_EQ(100.0 / memory_size_bytes, filter().get_relative_transient_memory_usage()); - EXPECT_EQ(200, filter().get_transient_disk_usage()); - EXPECT_EQ(200.0 / disk_size_bytes, filter().get_relative_transient_disk_usage()); + EXPECT_EQ(150, filter().get_transient_resource_usage().memory()); + EXPECT_EQ(150.0 / memory_size_bytes, filter().usageState().transient_memory_usage()); + EXPECT_EQ(350, filter().get_transient_resource_usage().disk()); + EXPECT_EQ(350.0 / disk_size_bytes, filter().usageState().transient_disk_usage()); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp index 01b46dd365f..7ff59a9f41a 100644 --- a/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp +++ b/searchcore/src/tests/proton/server/memory_flush_config_updater/memory_flush_config_updater_test.cpp @@ -205,12 +205,13 @@ TEST_F("require that we must go below low watermark for memory usage before usin TEST_F("require that more disk bloat is allowed while node state is retired", Fixture) { + constexpr double DEFAULT_DISK_BLOAT = 0.25; f.notifyDiskMemUsage(ResourceUsageState(0.7, 0.3), belowLimit()); - TEST_DO(f.assertStrategyDiskConfig(0.2, 0.2)); + TEST_DO(f.assertStrategyDiskConfig(DEFAULT_DISK_BLOAT, DEFAULT_DISK_BLOAT)); f.setNodeRetired(true); - TEST_DO(f.assertStrategyDiskConfig((0.8 - ((0.3/0.7)*(1 - 0.2))) / 0.8, 1.0)); + TEST_DO(f.assertStrategyDiskConfig((0.8 - ((0.3/0.7)*(1 - DEFAULT_DISK_BLOAT))) / 0.8, 1.0)); f.notifyDiskMemUsage(belowLimit(), belowLimit()); - TEST_DO(f.assertStrategyDiskConfig(0.2, 0.2)); + TEST_DO(f.assertStrategyDiskConfig(DEFAULT_DISK_BLOAT, DEFAULT_DISK_BLOAT)); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp b/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp index e90bfc8ae57..bede9f967a4 100644 --- a/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp +++ b/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp @@ -24,7 +24,7 @@ make_proton_config(double concurrency) builder.feeding.concurrency = concurrency; builder.feeding.sharedFieldWriterExecutor = ProtonConfig::Feeding::SharedFieldWriterExecutor::DOCUMENT_DB; - builder.indexing.tasklimit = 300; + builder.indexing.tasklimit = 255; return builder; } @@ -73,7 +73,8 @@ TEST_F(SharedThreadingServiceTest, field_writer_can_be_shared_across_all_documen setup(0.75, 8); EXPECT_TRUE(field_writer()); EXPECT_EQ(6, field_writer()->getNumExecutors()); - EXPECT_EQ(300, field_writer()->first_executor()->getTaskLimit()); + // This is rounded to the nearest power of 2 when using THROUGHPUT feed executor. + EXPECT_EQ(256, field_writer()->first_executor()->getTaskLimit()); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 7e99c1a19aa..34530afe9f9 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -54,7 +54,7 @@ flush.strategy enum {SIMPLE, MEMORY} default=MEMORY restart flush.memory.maxmemory long default=4294967296 ## Maximum total disk bloat factor before forcing flush. -flush.memory.diskbloatfactor double default=0.2 +flush.memory.diskbloatfactor double default=0.25 ## Max disk usage (in bytes) for all transaction logs before running flush. ## In this case the oldest component is flushed such that transaction log can be pruned and disk freed. @@ -65,7 +65,7 @@ flush.memory.maxtlssize long default=21474836480 flush.memory.each.maxmemory long default=1073741824 ## Maximum disk bloat factor per component before forcing flush. -flush.memory.each.diskbloatfactor double default=0.2 +flush.memory.each.diskbloatfactor double default=0.25 ## Age of unflushed content before forcing age prioritization. ## Unit is seconds with 31 hours being the default. @@ -130,7 +130,7 @@ indexing.threads int default=1 restart ## Option to specify what is most important during indexing. ## This is experimental and will most likely be temporary. -indexing.optimize enum {LATENCY, THROUGHPUT, ADAPTIVE} default=LATENCY restart +indexing.optimize enum {LATENCY, THROUGHPUT, ADAPTIVE} default=THROUGHPUT restart ## Maximum number of pending operations for each of the internal ## indexing threads. Only used when visibility delay is zero. @@ -203,7 +203,7 @@ distribution.searchablecopies long default=1 ## Control cache size in bytes. ## Postive numbers are absolute in bytes. ## Negative numbers are a percentage of memory. -summary.cache.maxbytes long default=-5 +summary.cache.maxbytes long default=-4 ## Include visits in the cache, if the visitoperation allows it. ## This will enable another separate cache of summary.cache.maxbytes size. @@ -517,7 +517,7 @@ feeding.shared_field_writer_executor enum {NONE, INDEX, INDEX_AND_ATTRIBUTE, DOC ## This limit is only considered when executing tasks for handling external feed operations. ## In that case the calling thread (persistence thread) is blocked until the master thread has capacity to handle more tasks. ## When this limit is set to 0 it is ignored. -feeding.master_task_limit int default = 0 +feeding.master_task_limit int default = 1000 ## Adjustment to resource limit when determining if maintenance jobs can run. ## diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp index 08432916d2b..16a4357d9f8 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.cpp @@ -2,37 +2,28 @@ #include "attribute_usage_sampler_context.h" #include "attribute_usage_filter.h" -#include <vespa/searchcore/proton/common/transient_resource_usage_provider.h> namespace proton { -AttributeUsageSamplerContext::AttributeUsageSamplerContext(AttributeUsageFilter& filter, - std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider) +AttributeUsageSamplerContext::AttributeUsageSamplerContext(AttributeUsageFilter& filter) : _usage(), - _transient_memory_usage(0u), _lock(), - _filter(filter), - _attribute_config_inspector(std::move(attribute_config_inspector)), - _transient_usage_provider(std::move(transient_usage_provider)) + _filter(filter) { } AttributeUsageSamplerContext::~AttributeUsageSamplerContext() { _filter.setAttributeStats(_usage); - _transient_usage_provider->set_transient_memory_usage(_transient_memory_usage); } void AttributeUsageSamplerContext::merge(const search::AddressSpaceUsage &usage, - size_t transient_memory_usage, const vespalib::string &attributeName, const vespalib::string &subDbName) { Guard guard(_lock); _usage.merge(usage, attributeName, subDbName); - _transient_memory_usage = std::max(_transient_memory_usage, transient_memory_usage); } -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h index 71cdc0e7f1a..bc43fb1ce4b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_context.h @@ -13,9 +13,8 @@ class AttributeConfigInspector; class TransientResourceUsageProvider; /* - * Context for sampling attribute usage stats and transient memory usage. - * When instance is destroyed, the aggregated stats is passed on to - * attribute usage filter and the transient memory usage provider. + * Context for sampling attribute usage stats. + * When instance is destroyed, the aggregated stats is passed on to attribute usage filter. */ class AttributeUsageSamplerContext { @@ -23,23 +22,16 @@ class AttributeUsageSamplerContext using Guard = std::lock_guard<Mutex>; AttributeUsageStats _usage; - size_t _transient_memory_usage; Mutex _lock; AttributeUsageFilter &_filter; - std::shared_ptr<const AttributeConfigInspector> _attribute_config_inspector; - std::shared_ptr<TransientResourceUsageProvider> _transient_usage_provider; + public: - AttributeUsageSamplerContext(AttributeUsageFilter& filter, - std::shared_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider); + AttributeUsageSamplerContext(AttributeUsageFilter& filter); ~AttributeUsageSamplerContext(); void merge(const search::AddressSpaceUsage &usage, - size_t transient_memory_usage, const vespalib::string &attributeName, const vespalib::string &subDbName); - const AttributeConfigInspector& get_attribute_config_inspector() const { return *_attribute_config_inspector; } - const AttributeUsageStats & - getUsage() const { return _usage; } + const AttributeUsageStats& getUsage() const { return _usage; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp index 6db20ab82a6..0af77f3e8cf 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.cpp @@ -3,7 +3,6 @@ #include "attribute_usage_sampler_functor.h" #include "attribute_usage_sampler_context.h" #include "attribute_config_inspector.h" -#include "attribute_transient_memory_calculator.h" #include <vespa/searchlib/attribute/attributevector.h> using search::attribute::BasicType; @@ -27,14 +26,7 @@ AttributeUsageSamplerFunctor::operator()(const search::attribute::IAttributeVect const auto & attributeVector = dynamic_cast<const search::AttributeVector &>(iAttributeVector); search::AddressSpaceUsage usage = attributeVector.getAddressSpaceUsage(); vespalib::string attributeName = attributeVector.getName(); - auto& old_config = attributeVector.getConfig(); - auto* current_config = _samplerContext->get_attribute_config_inspector().get_config(attributeName); - if (current_config == nullptr) { - current_config = &old_config; - } - AttributeTransientMemoryCalculator get_transient_memory_usage; - size_t transient_memory_usage = get_transient_memory_usage(attributeVector, *current_config); - _samplerContext->merge(usage, transient_memory_usage, attributeName, _subDbName); + _samplerContext->merge(usage, attributeName, _subDbName); } -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h index 963bd758494..d002173b205 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_usage_sampler_functor.h @@ -9,9 +9,8 @@ namespace proton { class AttributeUsageSamplerContext; -/* - * Functor for sampling attribute usage and passing it on to sampler - * context. +/** + * Functor for sampling attribute usage and passing it on to sampler context. */ class AttributeUsageSamplerFunctor : public search::attribute::IConstAttributeFunctor { diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index 219a2ea43a4..421e602a9cf 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -23,7 +23,6 @@ vespa_add_library(searchcore_pcommon STATIC selectpruner.cpp state_reporter_utils.cpp statusreport.cpp - transient_resource_usage_provider.cpp DEPENDS searchcore_proton_metrics searchcore_fconfig diff --git a/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h index fd50f46241c..833b6519046 100644 --- a/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h +++ b/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h @@ -7,16 +7,41 @@ namespace proton { /** - * Interface class providing transient resource usage. + * Class containing transient disk and memory usage (in bytes). + */ +class TransientResourceUsage { +private: + size_t _disk; + size_t _memory; + +public: + TransientResourceUsage() noexcept + : _disk(0), + _memory(0) + {} + TransientResourceUsage(size_t disk_in, + size_t memory_in) noexcept + : _disk(disk_in), + _memory(memory_in) + {} + size_t disk() const noexcept { return _disk; } + size_t memory() const noexcept { return _memory; } + void merge(const TransientResourceUsage& rhs) { + _disk += rhs.disk(); + _memory += rhs.memory(); + } +}; + +/** + * Interface class providing a snapshot of transient resource usage. * - * E.g. extra memory needed for loading or saving an attribute vectors or extra disk needed for running disk index fusion. - * It provides an aggregated max view over several components (e.g. all attribute vectors for a document type). + * E.g. the memory used by the memory index and extra disk needed for running disk index fusion. + * This provides the total transient resource usage for the components this provider encapsulates. */ class ITransientResourceUsageProvider { public: virtual ~ITransientResourceUsageProvider() = default; - virtual size_t get_transient_memory_usage() const = 0; - virtual size_t get_transient_disk_usage() const = 0; + virtual TransientResourceUsage get_transient_resource_usage() const = 0; }; } diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.cpp b/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.cpp deleted file mode 100644 index 0d338cad9df..00000000000 --- a/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "transient_resource_usage_provider.h" - -namespace proton { - -TransientResourceUsageProvider::TransientResourceUsageProvider() - : ITransientResourceUsageProvider(), - _transient_memory_usage(0u) -{ -} - -TransientResourceUsageProvider::~TransientResourceUsageProvider() = default; - -size_t -TransientResourceUsageProvider::get_transient_memory_usage() const -{ - return _transient_memory_usage.load(std::memory_order_relaxed); -} - -void -TransientResourceUsageProvider::set_transient_memory_usage(size_t transient_memory_usage) -{ - _transient_memory_usage.store(transient_memory_usage, std::memory_order_relaxed); -} - -} diff --git a/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.h deleted file mode 100644 index 205e95d9fa6..00000000000 --- a/searchcore/src/vespa/searchcore/proton/common/transient_resource_usage_provider.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "i_transient_resource_usage_provider.h" - -#include <atomic> - -namespace proton { - -/** - * Class providing transient resource usage. - * E.g. extra memory needed for loading or saving an attribute vector. - * It provides an aggregated view over several components (e.g. all attribute vectors for a document type). - */ -class TransientResourceUsageProvider : public ITransientResourceUsageProvider { - std::atomic<size_t> _transient_memory_usage; -public: - TransientResourceUsageProvider(); - virtual ~TransientResourceUsageProvider(); - size_t get_transient_memory_usage() const override; - size_t get_transient_disk_usage() const override { return 0; } - void set_transient_memory_usage(size_t transient_memory_usage); -}; - -} diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp index bfca455217c..251d0475537 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp @@ -69,9 +69,9 @@ IndexManager::MaintainerOperations::runFusion(const Schema &schema, std::shared_ptr<IFlushToken> flush_token) { SerialNumFileHeaderContext fileHeaderContext(_fileHeaderContext, serialNum); - const bool dynamic_k_doc_pos_occ_format = false; - return Fusion::merge(schema, outputDir, sources, selectorArray, dynamic_k_doc_pos_occ_format, - _tuneFileIndexing, fileHeaderContext, _threadingService.shared(), std::move(flush_token)); + Fusion fusion(schema, outputDir, sources, selectorArray, + _tuneFileIndexing, fileHeaderContext); + return fusion.merge(_threadingService.shared(), std::move(flush_token)); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp index b39b5dc5734..278b0c68dab 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "document_iterator.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/searchcore/proton/common/cachedselect.h> #include <vespa/searchcore/proton/common/selectcontext.h> #include <vespa/document/select/gid_filter.h> @@ -8,7 +9,6 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/stllike/hash_map.h> -#include <algorithm> #include <vespa/log/log.h> LOG_SETUP(".proton.persistenceengine.document_iterator"); @@ -18,23 +18,25 @@ using storage::spi::DocEntry; using storage::spi::Timestamp; using document::Document; using document::DocumentId; +using storage::spi::DocumentMetaEnum; namespace proton { namespace { -DocEntry *createDocEntry(Timestamp timestamp, bool removed) { - int flags = removed ? storage::spi::REMOVE_ENTRY : storage::spi::NONE; - return new DocEntry(timestamp, flags); +std::unique_ptr<DocEntry> +createDocEntry(Timestamp timestamp, bool removed) { + return DocEntry::create(timestamp, removed ? DocumentMetaEnum::REMOVE_ENTRY : DocumentMetaEnum::NONE); } -DocEntry *createDocEntry(Timestamp timestamp, bool removed, Document::UP doc, ssize_t defaultSerializedSize) { +std::unique_ptr<DocEntry> +createDocEntry(Timestamp timestamp, bool removed, Document::UP doc, ssize_t defaultSerializedSize) { if (doc) { if (removed) { - return new DocEntry(timestamp, storage::spi::REMOVE_ENTRY, doc->getId()); + return DocEntry::create(timestamp, DocumentMetaEnum::REMOVE_ENTRY, doc->getId()); } else { ssize_t serializedSize = defaultSerializedSize >= 0 ? defaultSerializedSize : doc->serialize().size(); - return new DocEntry(timestamp, storage::spi::NONE, std::move(doc), serializedSize); + return DocEntry::create(timestamp, std::move(doc), serializedSize); } } else { return createDocEntry(timestamp, removed); @@ -212,7 +214,7 @@ public: if (doc && _fields) { document::FieldSet::stripFields(*doc, *_fields); } - _list.emplace_back(createDocEntry(meta.timestamp, meta.removed, std::move(doc), _defaultSerializedSize)); + _list.push_back(createDocEntry(meta.timestamp, meta.removed, std::move(doc), _defaultSerializedSize)); } } @@ -262,11 +264,12 @@ DocumentIterator::fetchCompleteSource(const IDocumentRetriever & source, Iterate } LOG(debug, "metadata count after filtering: %zu", lidsToFetch.size()); + list.reserve(lidsToFetch.size()); if ( _metaOnly ) { for (uint32_t lid : lidsToFetch) { const search::DocumentMetaData & meta = metaData[lidIndexMap[lid]]; assert(lid == meta.lid); - list.emplace_back(createDocEntry(meta.timestamp, meta.removed)); + list.push_back(createDocEntry(meta.timestamp, meta.removed)); } } else { MatchVisitor visitor(matcher, metaData, lidIndexMap, _fields.get(), list, _defaultSerializedSize); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp index d359583cb59..6307604598d 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/resource_usage_tracker.cpp @@ -94,6 +94,7 @@ void ResourceUsageTracker::notifyDiskMemUsage(DiskMemUsageState state) { std::lock_guard guard(_lock); + // TODO: Subtract transient resource (memory and disk) usage from the absolute numbers. _resource_usage = ResourceUsage(state.diskState().usage(), state.memoryState().usage(), _resource_usage.get_attribute_address_space_usage()); if (_listener != nullptr) { _listener->update_resource_usage(_resource_usage); diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index 3677b9165fb..8928045b814 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -130,34 +130,46 @@ DiskMemUsageFilter::recalcState(const Guard &guard) _acceptWrite = true; } DiskMemUsageState dmstate(ResourceUsageState(_config._diskLimit, diskUsed), - ResourceUsageState(_config._memoryLimit, memoryUsed)); + ResourceUsageState(_config._memoryLimit, memoryUsed), + get_relative_transient_disk_usage(guard), + get_relative_transient_memory_usage(guard)); notifyDiskMemUsage(guard, dmstate); } double -DiskMemUsageFilter::getMemoryUsedRatio(const Guard &guard) const +DiskMemUsageFilter::getMemoryUsedRatio(const Guard&) const { - (void) guard; uint64_t unscaledMemoryUsed = _memoryStats.getAnonymousRss(); return static_cast<double>(unscaledMemoryUsed) / _hwInfo.memory().sizeBytes(); } double -DiskMemUsageFilter::getDiskUsedRatio(const Guard &guard) const +DiskMemUsageFilter::getDiskUsedRatio(const Guard&) const { - (void) guard; double usedDiskSpaceRatio = static_cast<double>(_diskUsedSizeBytes) / static_cast<double>(_hwInfo.disk().sizeBytes()); return usedDiskSpaceRatio; } +double +DiskMemUsageFilter::get_relative_transient_memory_usage(const Guard&) const +{ + return static_cast<double>(_transient_usage.memory()) / _hwInfo.memory().sizeBytes(); +} + +double +DiskMemUsageFilter::get_relative_transient_disk_usage(const Guard&) const +{ + return static_cast<double>(_transient_usage.disk()) / _hwInfo.disk().sizeBytes(); +} + DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo) : _lock(), _hwInfo(hwInfo), _acceptWrite(true), _memoryStats(), _diskUsedSizeBytes(), - _transient_memory_usage(0u), + _transient_usage(), _config(), _state(), _dmstate(), @@ -184,11 +196,11 @@ DiskMemUsageFilter::setDiskUsedSize(uint64_t diskUsedSizeBytes) } void -DiskMemUsageFilter::set_transient_resource_usage(size_t transient_memory_usage, size_t transient_disk_usage) +DiskMemUsageFilter::set_transient_resource_usage(const TransientResourceUsage& transient_usage) { Guard guard(_lock); - _transient_memory_usage = transient_memory_usage; - _transient_disk_usage = transient_disk_usage; + _transient_usage = transient_usage; + recalcState(guard); } void @@ -213,32 +225,11 @@ DiskMemUsageFilter::getDiskUsedSize() const return _diskUsedSizeBytes; } -size_t -DiskMemUsageFilter::get_transient_memory_usage() const -{ - Guard guard(_lock); - return _transient_memory_usage; -} - -double -DiskMemUsageFilter::get_relative_transient_memory_usage() const -{ - Guard guard(_lock); - return static_cast<double>(_transient_memory_usage) / _hwInfo.memory().sizeBytes(); -} - -size_t -DiskMemUsageFilter::get_transient_disk_usage() const -{ - Guard guard(_lock); - return _transient_disk_usage; -} - -double -DiskMemUsageFilter::get_relative_transient_disk_usage() const +TransientResourceUsage +DiskMemUsageFilter::get_transient_resource_usage() const { Guard guard(_lock); - return static_cast<double>(_transient_disk_usage) / _hwInfo.disk().sizeBytes(); + return _transient_usage; } DiskMemUsageFilter::Config diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h index e2e94b1abd9..ac328c3ddea 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h @@ -6,6 +6,7 @@ #include "disk_mem_usage_state.h" #include "disk_mem_usage_metrics.h" #include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/searchcore/proton/common/i_transient_resource_usage_provider.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> #include <vespa/vespalib/util/process_memory_stats.h> #include <atomic> @@ -15,9 +16,9 @@ namespace proton { -/* - * Class to filter write operations based on sampled disk and memory - * usage. If resource limit is reached then further writes are denied +/** + * Class to filter write operations based on sampled disk and memory usage. + * If resource limit is reached then further writes are denied * in order to prevent entering an unrecoverable state. */ class DiskMemUsageFilter : public IResourceWriteFilter, @@ -47,8 +48,7 @@ private: // Following member variables are protected by _lock vespalib::ProcessMemoryStats _memoryStats; uint64_t _diskUsedSizeBytes; - size_t _transient_memory_usage; - size_t _transient_disk_usage; + TransientResourceUsage _transient_usage; Config _config; State _state; DiskMemUsageState _dmstate; @@ -58,6 +58,8 @@ private: void recalcState(const Guard &guard); // called with _lock held double getMemoryUsedRatio(const Guard &guard) const; double getDiskUsedRatio(const Guard &guard) const; + double get_relative_transient_memory_usage(const Guard& guard) const; + double get_relative_transient_disk_usage(const Guard& guard) const; void notifyDiskMemUsage(const Guard &guard, DiskMemUsageState state); public: @@ -65,14 +67,11 @@ public: ~DiskMemUsageFilter() override; void setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in); void setDiskUsedSize(uint64_t diskUsedSizeBytes); - void set_transient_resource_usage(size_t transient_memory_usage, size_t transient_disk_usage); + void set_transient_resource_usage(const TransientResourceUsage& transient_usage); void setConfig(Config config); vespalib::ProcessMemoryStats getMemoryStats() const; uint64_t getDiskUsedSize() const; - size_t get_transient_memory_usage() const; - double get_relative_transient_memory_usage() const; - size_t get_transient_disk_usage() const; - double get_relative_transient_disk_usage() const; + TransientResourceUsage get_transient_resource_usage() const; Config getConfig() const; const HwInfo &getHwInfo() const { return _hwInfo; } DiskMemUsageState usageState() const; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp index 60f45f536d6..230593c2c1d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp @@ -14,8 +14,10 @@ DiskMemUsageMetrics::DiskMemUsageMetrics() noexcept DiskMemUsageMetrics::DiskMemUsageMetrics(const DiskMemUsageState &usage_state) noexcept : _disk_usage(usage_state.diskState().usage()), _disk_utilization(usage_state.diskState().utilization()), + _transient_disk_usage(usage_state.transient_disk_usage()), _memory_usage(usage_state.memoryState().usage()), - _memory_utilization(usage_state.memoryState().utilization()) + _memory_utilization(usage_state.memoryState().utilization()), + _transient_memory_usage(usage_state.transient_memory_usage()) { } @@ -24,8 +26,10 @@ DiskMemUsageMetrics::merge(const DiskMemUsageState &usage_state) noexcept { _disk_usage = std::max(_disk_usage, usage_state.diskState().usage()); _disk_utilization = std::max(_disk_utilization, usage_state.diskState().utilization()); + _transient_disk_usage = std::max(_transient_disk_usage, usage_state.transient_disk_usage()); _memory_usage = std::max(_memory_usage, usage_state.memoryState().usage()); _memory_utilization = std::max(_memory_utilization, usage_state.memoryState().utilization()); + _transient_memory_usage = std::max(_transient_memory_usage, usage_state.transient_memory_usage()); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h index 10687fd38a8..cb97eb4c891 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h @@ -14,8 +14,10 @@ class DiskMemUsageMetrics { double _disk_usage; double _disk_utilization; + double _transient_disk_usage; double _memory_usage; double _memory_utilization; + double _transient_memory_usage; public: DiskMemUsageMetrics() noexcept; @@ -23,8 +25,10 @@ public: void merge(const DiskMemUsageState &usage_state) noexcept; double get_disk_usage() const noexcept { return _disk_usage; } double get_disk_utilization() const noexcept { return _disk_utilization; } + double get_transient_disk_usage() const noexcept { return _transient_disk_usage; } double get_memory_usage() const noexcept { return _memory_usage; } double get_memory_utilization() const noexcept { return _memory_utilization; } + double get_transient_memory_usage() const noexcept { return _transient_memory_usage; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp index d9f47c2944b..2dc749ce26b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp @@ -125,16 +125,14 @@ DiskMemUsageSampler::sampleMemoryUsage() void DiskMemUsageSampler::sample_transient_resource_usage() { - size_t max_transient_memory_usage = 0; - size_t max_transient_disk_usage = 0; + TransientResourceUsage transient_usage; { std::lock_guard<std::mutex> guard(_lock); for (auto provider : _transient_usage_providers) { - max_transient_memory_usage = std::max(max_transient_memory_usage, provider->get_transient_memory_usage()); - max_transient_disk_usage = std::max(max_transient_disk_usage, provider->get_transient_disk_usage()); + transient_usage.merge(provider->get_transient_resource_usage()); } } - _filter.set_transient_resource_usage(max_transient_memory_usage, max_transient_disk_usage); + _filter.set_transient_resource_usage(transient_usage); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h index 7e1314e134d..b205b441bcf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h @@ -8,31 +8,40 @@ namespace proton { /** * Class used to describe state of disk and memory usage relative to configured limits. + * In addition relative transient disk and memory usage are tracked. */ class DiskMemUsageState { ResourceUsageState _diskState; ResourceUsageState _memoryState; + double _transient_disk_usage; + double _transient_memory_usage; public: DiskMemUsageState() = default; DiskMemUsageState(const ResourceUsageState &diskState_, - const ResourceUsageState &memoryState_) + const ResourceUsageState &memoryState_, + double transient_disk_usage_ = 0, + double transient_memory_usage_ = 0) : _diskState(diskState_), - _memoryState(memoryState_) + _memoryState(memoryState_), + _transient_disk_usage(transient_disk_usage_), + _transient_memory_usage(transient_memory_usage_) { } bool operator==(const DiskMemUsageState &rhs) const { return ((_diskState == rhs._diskState) && - (_memoryState == rhs._memoryState)); + (_memoryState == rhs._memoryState) && + (_transient_disk_usage == rhs._transient_disk_usage) && + (_transient_memory_usage == rhs._transient_memory_usage)); } bool operator!=(const DiskMemUsageState &rhs) const { return ! ((*this) == rhs); } const ResourceUsageState &diskState() const { return _diskState; } const ResourceUsageState &memoryState() const { return _memoryState; } - bool aboveDiskLimit() const { return diskState().aboveLimit(); } - bool aboveMemoryLimit() const { return memoryState().aboveLimit(); } + double transient_disk_usage() const { return _transient_disk_usage; } + double transient_memory_usage() const { return _transient_memory_usage; } bool aboveDiskLimit(double resourceLimitFactor) const { return diskState().aboveLimit(resourceLimitFactor); } bool aboveMemoryLimit(double resourceLimitFactor) const { return memoryState().aboveLimit(resourceLimitFactor); } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 53bdc356015..f052d663ba6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -19,8 +19,8 @@ #include <vespa/searchcore/proton/attribute/i_attribute_usage_listener.h> #include <vespa/searchcore/proton/attribute/imported_attributes_repo.h> #include <vespa/searchcore/proton/common/eventlogger.h> +#include <vespa/searchcore/proton/common/i_transient_resource_usage_provider.h> #include <vespa/searchcore/proton/common/statusreport.h> -#include <vespa/searchcore/proton/common/transient_resource_usage_provider.h> #include <vespa/searchcore/proton/docsummary/isummarymanager.h> #include <vespa/searchcore/proton/feedoperation/noopoperation.h> #include <vespa/searchcore/proton/index/index_writer.h> @@ -89,18 +89,19 @@ public: } }; -class DocumentDBResourceUsageProvider : public TransientResourceUsageProvider { +class DocumentDBResourceUsageProvider : public ITransientResourceUsageProvider { private: const DocumentDB& _doc_db; public: - DocumentDBResourceUsageProvider(const DocumentDB& doc_db) + DocumentDBResourceUsageProvider(const DocumentDB& doc_db) noexcept : _doc_db(doc_db) {} - size_t get_transient_disk_usage() const override { - // We estimate the transient disk usage for the next disk index fusion - // as the size of the largest disk index. - return _doc_db.getReadySubDB()->getSearchableStats().max_component_size_on_disk(); + TransientResourceUsage get_transient_resource_usage() const override { + // Transient disk usage is measured as the total disk usage of all current fusion indexes. + // Transient memory usage is measured as the total memory usage of all memory indexes. + auto stats = _doc_db.getReadySubDB()->getSearchableStats(); + return {stats.fusion_size_on_disk(), stats.memoryUsage().allocatedBytes()}; } }; @@ -909,7 +910,7 @@ DocumentDB::hasDocument(const document::DocumentId &id) } void -DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector) +DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config) { // Called by executor thread _maintenanceController.killJobs(); @@ -931,8 +932,6 @@ DocumentDB::injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std _jobTrackers, _subDBs.getReadySubDB()->getAttributeManager(), _subDBs.getNotReadySubDB()->getAttributeManager(), - std::move(attribute_config_inspector), - _transient_usage_provider, _writeFilter); } @@ -954,9 +953,7 @@ DocumentDB::performStartMaintenance() return; } auto maintenanceConfig = activeConfig->getMaintenanceConfigSP(); - const auto &attributes_config = activeConfig->getAttributesConfig(); - auto attribute_config_inspector = std::make_unique<AttributeConfigInspector>(attributes_config); - injectMaintenanceJobs(*maintenanceConfig, std::move(attribute_config_inspector)); + injectMaintenanceJobs(*maintenanceConfig); _maintenanceController.start(maintenanceConfig); } @@ -973,11 +970,9 @@ DocumentDB::forwardMaintenanceConfig() DocumentDBConfig::SP activeConfig = getActiveConfig(); assert(activeConfig); auto maintenanceConfig(activeConfig->getMaintenanceConfigSP()); - const auto &attributes_config = activeConfig->getAttributesConfig(); - auto attribute_config_inspector = std::make_unique<AttributeConfigInspector>(attributes_config); if (!_state.getClosed()) { if (_maintenanceController.getPaused()) { - injectMaintenanceJobs(*maintenanceConfig, std::move(attribute_config_inspector)); + injectMaintenanceJobs(*maintenanceConfig); } _maintenanceController.newConfig(maintenanceConfig); } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index e829f477e8a..8e8391b2f31 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -52,7 +52,6 @@ class IDocumentDBOwner; class ISharedThreadingService; class ITransientResourceUsageProvider; class StatusReport; -class TransientResourceUsageProvider; struct MetricsWireService; namespace matching { class SessionManager; } @@ -117,7 +116,7 @@ private: DDBState _state; DiskMemUsageForwarder _dmUsageForwarder; AttributeUsageFilter _writeFilter; - std::shared_ptr<TransientResourceUsageProvider> _transient_usage_provider; + std::shared_ptr<ITransientResourceUsageProvider> _transient_usage_provider; std::unique_ptr<FeedHandler> _feedHandler; DocumentSubDBCollection _subDBs; MaintenanceController _maintenanceController; @@ -384,7 +383,7 @@ public: /** * Implements IFeedHandlerOwner **/ - void injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config, std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector); + void injectMaintenanceJobs(const DocumentDBMaintenanceConfig &config); void performStartMaintenance(); void stopMaintenance(); void forwardMaintenanceConfig(); diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index a832f00f911..f4b92876891 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -9,7 +9,6 @@ #include "prune_session_cache_job.h" #include "pruneremoveddocumentsjob.h" #include "sample_attribute_usage_job.h" -#include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> using vespalib::system_clock; @@ -87,8 +86,6 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, DocumentDBJobTrackers &jobTrackers, IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, - std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider, AttributeUsageFilter &attributeUsageFilter) { controller.registerJobInMasterThread(std::make_unique<HeartBeatJob>(hbHandler, config.getHeartBeatConfig())); @@ -122,9 +119,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, controller.registerJobInMasterThread( std::make_unique<SampleAttributeUsageJob>(readyAttributeManager, notReadyAttributeManager, attributeUsageFilter, docTypeName, - config.getAttributeUsageSampleInterval(), - std::move(attribute_config_inspector), - transient_usage_provider)); + config.getAttributeUsageSampleInterval())); } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h index 8d0b01465fb..1dcfafaa645 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.h @@ -12,17 +12,15 @@ namespace storage::spi {struct BucketExecutor; } namespace proton { -class AttributeConfigInspector; -class IPruneRemovedDocumentsHandler; -struct IDocumentMoveHandler; +class AttributeUsageFilter; class IBucketModifiedHandler; -class IClusterStateChangedNotifier; class IBucketStateChangedNotifier; -struct IBucketStateCalculator; -struct IAttributeManager; -class AttributeUsageFilter; +class IClusterStateChangedNotifier; class IDiskMemUsageNotifier; -class TransientResourceUsageProvider; +class IPruneRemovedDocumentsHandler; +struct IAttributeManager; +struct IBucketStateCalculator; +struct IDocumentMoveHandler; namespace bucketdb { class IBucketCreateNotifier; } /** @@ -50,8 +48,6 @@ struct MaintenanceJobsInjector DocumentDBJobTrackers &jobTrackers, IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, - std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider, AttributeUsageFilter &attributeUsageFilter); }; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index f697c4d4672..b128fe16e5e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -757,8 +757,8 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) metrics.resourceUsage.diskUtilization.set(dm_metrics.get_disk_utilization()); metrics.resourceUsage.memory.set(dm_metrics.get_memory_usage()); metrics.resourceUsage.memoryUtilization.set(dm_metrics.get_memory_utilization()); - metrics.resourceUsage.transient_memory.set(usageFilter.get_relative_transient_memory_usage()); - metrics.resourceUsage.transient_disk.set(usageFilter.get_relative_transient_disk_usage()); + metrics.resourceUsage.transient_memory.set(dm_metrics.get_transient_memory_usage()); + metrics.resourceUsage.transient_disk.set(dm_metrics.get_transient_disk_usage()); metrics.resourceUsage.memoryMappings.set(usageFilter.getMemoryStats().getMappingsCount()); metrics.resourceUsage.openFileDescriptors.set(FastOS_File::count_open_files()); metrics.resourceUsage.feedingBlocked.set((usageFilter.acceptWriteOperation() ? 0.0 : 1.0)); diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp index 4b5e97a827b..3750c5afb5f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp @@ -44,16 +44,16 @@ ResourceUsageExplorer::get_state(const vespalib::slime::Inserter &inserter, bool disk.setDouble("usage", usageState.diskState().usage()); disk.setDouble("limit", usageState.diskState().limit()); disk.setDouble("utilization", usageState.diskState().utilization()); + disk.setDouble("transient", usageState.transient_disk_usage()); convertDiskStatsToSlime(_usage_filter.getHwInfo(), _usage_filter.getDiskUsedSize(), disk.setObject("stats")); Cursor &memory = object.setObject("memory"); memory.setDouble("usage", usageState.memoryState().usage()); memory.setDouble("limit", usageState.memoryState().limit()); memory.setDouble("utilization", usageState.memoryState().utilization()); + memory.setDouble("transient", usageState.transient_memory_usage()); memory.setLong("physicalMemory", _usage_filter.getHwInfo().memory().sizeBytes()); convertMemoryStatsToSlime(_usage_filter.getMemoryStats(), memory.setObject("stats")); - size_t transient_memory = _usage_filter.get_transient_memory_usage(); - memory.setLong("transient", transient_memory); Cursor &address_space = object.setObject("attribute_address_space"); address_space.setDouble("usage", attr_usage.get_usage()); diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp index 7dc96b92de9..e21e0366c4c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp @@ -14,15 +14,11 @@ SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, AttributeUsageFilter &attributeUsageFilter, const vespalib::string &docTypeName, - vespalib::duration interval, - std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider) + vespalib::duration interval) : IMaintenanceJob("sample_attribute_usage." + docTypeName, vespalib::duration::zero(), interval), _readyAttributeManager(readyAttributeManager), _notReadyAttributeManager(notReadyAttributeManager), - _attributeUsageFilter(attributeUsageFilter), - _attribute_config_inspector(std::move(attribute_config_inspector)), - _transient_usage_provider(std::move(transient_usage_provider)) + _attributeUsageFilter(attributeUsageFilter) { } @@ -31,7 +27,7 @@ SampleAttributeUsageJob::~SampleAttributeUsageJob() = default; bool SampleAttributeUsageJob::run() { - auto context = std::make_shared<AttributeUsageSamplerContext> (_attributeUsageFilter, _attribute_config_inspector, _transient_usage_provider); + auto context = std::make_shared<AttributeUsageSamplerContext> (_attributeUsageFilter); _readyAttributeManager->asyncForEachAttribute(std::make_shared<AttributeUsageSamplerFunctor>(context, "ready")); _notReadyAttributeManager->asyncForEachAttribute(std::make_shared<AttributeUsageSamplerFunctor>(context, "notready")); return true; diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h index d72305496dd..9e42897f8e7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h +++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.h @@ -7,9 +7,7 @@ namespace proton { struct IAttributeManager; -class AttributeConfigInspector; class AttributeUsageFilter; -class TransientResourceUsageProvider; /** * Class used to sample attribute resource usage and pass aggregated @@ -23,16 +21,13 @@ class SampleAttributeUsageJob : public IMaintenanceJob IAttributeManagerSP _readyAttributeManager; IAttributeManagerSP _notReadyAttributeManager; AttributeUsageFilter &_attributeUsageFilter; - std::shared_ptr<const AttributeConfigInspector> _attribute_config_inspector; - std::shared_ptr<TransientResourceUsageProvider> _transient_usage_provider; + public: SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager, IAttributeManagerSP notReadyAttributeManager, AttributeUsageFilter &attributeUsageFilter, const vespalib::string &docTypeName, - vespalib::duration interval, - std::unique_ptr<const AttributeConfigInspector> attribute_config_inspector, - std::shared_ptr<TransientResourceUsageProvider> transient_usage_provider); + vespalib::duration interval); ~SampleAttributeUsageJob() override; bool run() override; diff --git a/searchcorespi/CMakeLists.txt b/searchcorespi/CMakeLists.txt index 66ff3fff0f2..0bbaa813752 100644 --- a/searchcorespi/CMakeLists.txt +++ b/searchcorespi/CMakeLists.txt @@ -18,4 +18,8 @@ vespa_define_module( src/vespa/searchcorespi src/vespa/searchcorespi/flush src/vespa/searchcorespi/index + + TESTS + src/tests/index/disk_indexes + src/tests/index/index_disk_layout ) diff --git a/searchcorespi/src/tests/index/disk_indexes/CMakeLists.txt b/searchcorespi/src/tests/index/disk_indexes/CMakeLists.txt new file mode 100644 index 00000000000..81b6bb0e8a9 --- /dev/null +++ b/searchcorespi/src/tests/index/disk_indexes/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcorespi_disk_indexes_test_app + SOURCES + disk_indexes_test.cpp + DEPENDS + searchcorespi + GTest::GTest +) +vespa_add_test(NAME searchcorespi_disk_indexes_test_app COMMAND searchcorespi_disk_indexes_test_app) diff --git a/searchcorespi/src/tests/index/disk_indexes/disk_indexes_test.cpp b/searchcorespi/src/tests/index/disk_indexes/disk_indexes_test.cpp new file mode 100644 index 00000000000..d22ad499316 --- /dev/null +++ b/searchcorespi/src/tests/index/disk_indexes/disk_indexes_test.cpp @@ -0,0 +1,196 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcorespi/index/disk_indexes.h> +#include <vespa/searchcorespi/index/index_disk_dir.h> +#include <vespa/searchcorespi/index/indexdisklayout.h> +#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <fstream> + +namespace { + +vespalib::string base_dir("base"); + +constexpr uint32_t block_size = 4_Ki; + +} + +namespace searchcorespi::index { + +class DiskIndexesTest : public ::testing::Test, + public DiskIndexes +{ + IndexDiskLayout _layout; +protected: + DiskIndexesTest(); + ~DiskIndexesTest(); + + static IndexDiskDir get_index_disk_dir(const vespalib::string& dir) { + return IndexDiskLayout::get_index_disk_dir(dir); + } + + void assert_transient_size(uint64_t exp, IndexDiskDir index_disk_dir) { + EXPECT_EQ(exp, get_transient_size(_layout, index_disk_dir)); + } +}; + +DiskIndexesTest::DiskIndexesTest() + : ::testing::Test(), + DiskIndexes(), + _layout(base_dir) +{ +} + +DiskIndexesTest::~DiskIndexesTest() = default; + +TEST_F(DiskIndexesTest, simple_set_active_works) +{ + EXPECT_FALSE(isActive("index.flush.1")); + setActive("index.flush.1", 0); + EXPECT_TRUE(isActive("index.flush.1")); + notActive("index.flush.1"); + EXPECT_FALSE(isActive("index.flush.1")); +} + +TEST_F(DiskIndexesTest, nested_set_active_works) +{ + setActive("index.flush.1", 0); + setActive("index.flush.1", 0); + EXPECT_TRUE(isActive("index.flush.1")); + notActive("index.flush.1"); + EXPECT_TRUE(isActive("index.flush.1")); + notActive("index.flush.1"); + EXPECT_FALSE(isActive("index.flush.1")); +} + +TEST_F(DiskIndexesTest, is_active_returns_false_for_bad_name) +{ + EXPECT_FALSE(isActive("foo/bar/baz")); + EXPECT_FALSE(isActive("index.flush.0")); +} + +TEST_F(DiskIndexesTest, remove_works) +{ + EXPECT_TRUE(remove(IndexDiskDir())); + auto fusion1 = get_index_disk_dir("index.fusion.1"); + EXPECT_TRUE(remove(fusion1)); + add_not_active(fusion1); + EXPECT_TRUE(remove(fusion1)); + setActive("index.fusion.1", 0); + EXPECT_FALSE(remove(fusion1)); + notActive("index.fusion.1"); + EXPECT_TRUE(remove(fusion1)); +} + +TEST_F(DiskIndexesTest, basic_get_transient_size_works) +{ + /* + * When starting to use a new fusion index, we have a transient + * period with two ISearchableIndexCollection instances: + * - old, containing index.fusion.1 and index.flush.2 + * - new, containing index.fusion.2 + */ + setActive("index.fusion.1", 1000000); + setActive("index.flush.2", 500000); + setActive("index.fusion.2", 1200000); + auto fusion1 = get_index_disk_dir("index.fusion.1"); + auto flush2 = get_index_disk_dir("index.flush.2"); + auto fusion2 = get_index_disk_dir("index.fusion.2"); + { + /* + * When using the old index collection, disk space used by + * index.fusion.2 is considered transient. + */ + SCOPED_TRACE("index.fusion.1"); + assert_transient_size(1200000, fusion1); + } + { + SCOPED_TRACE("index.flush.2"); + assert_transient_size(0, flush2); + } + { + /* + * When using the new index collection, disk space used by + * index.fusion.1 and index.flush.2 is considered transient. + */ + SCOPED_TRACE("index.fusion.2"); + assert_transient_size(1500000, fusion2); + } + notActive("index.fusion.1"); + notActive("index.flush.2"); + { + /* + * old index collection removed. + */ + SCOPED_TRACE("index.fusion.2 after remove of index.fusion.1 and index.flush.1"); + assert_transient_size(0, fusion2); + } +} + +TEST_F(DiskIndexesTest, get_transient_size_during_ongoing_fusion) +{ + /* + * During ongoing fusion, we have one ISearchableIndexCollection instance: + * - old, containing index.fusion.1 and index.flush.2 + * + * Fusion output directory is index.fusion.2 + */ + setActive("index.fusion.1", 1000000); + setActive("index.flush.2", 500000); + auto fusion1 = get_index_disk_dir("index.fusion.1"); + auto fusion2 = get_index_disk_dir("index.fusion.2"); + add_not_active(fusion2); // start tracking disk space for fusion output + { + /* + * Fusion not yet started. + */ + SCOPED_TRACE("dir missing"); + assert_transient_size(0, fusion1); + } + auto dir = base_dir + "/index.fusion.2"; + vespalib::mkdir(dir, true); + { + /* + * Fusion started, but no files written yet. + */ + SCOPED_TRACE("empty dir"); + assert_transient_size(0, fusion1); + } + constexpr uint32_t seek_pos = 999999; + { + std::string name = dir + "/foo"; + std::ofstream ostr(name, std::ios::binary); + ostr.seekp(seek_pos); + ostr.write(" ", 1); + ostr.flush(); + ostr.close(); + } + { + /* + * Fusion started, one file written. + */ + SCOPED_TRACE("single file"); + assert_transient_size((seek_pos + block_size) / block_size * block_size, fusion1); + } + EXPECT_TRUE(remove(fusion2)); // stop tracking disk space for fusion output + { + /* + * Fusion aborted. + */ + SCOPED_TRACE("removed"); + assert_transient_size(0, fusion1); + } +} + +} + +int +main(int argc, char* argv[]) +{ + vespalib::rmdir(base_dir, true); + ::testing::InitGoogleTest(&argc, argv); + auto result = RUN_ALL_TESTS(); + vespalib::rmdir(base_dir, true); + return result; +} diff --git a/searchcorespi/src/tests/index/index_disk_layout/CMakeLists.txt b/searchcorespi/src/tests/index/index_disk_layout/CMakeLists.txt new file mode 100644 index 00000000000..4e82cf1b9d2 --- /dev/null +++ b/searchcorespi/src/tests/index/index_disk_layout/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcorespi_index_disk_layout_test_app + SOURCES + index_disk_layout_test.cpp + DEPENDS + searchcorespi + GTest::GTest +) +vespa_add_test(NAME searchcorespi_index_disk_layout_test_app COMMAND searchcorespi_index_disk_layout_test_app) diff --git a/searchcorespi/src/tests/index/index_disk_layout/index_disk_layout_test.cpp b/searchcorespi/src/tests/index/index_disk_layout/index_disk_layout_test.cpp new file mode 100644 index 00000000000..e35225b2745 --- /dev/null +++ b/searchcorespi/src/tests/index/index_disk_layout/index_disk_layout_test.cpp @@ -0,0 +1,60 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/searchcorespi/index/indexdisklayout.h> +#include <vespa/searchcorespi/index/index_disk_dir.h> +#include <vespa/vespalib/gtest/gtest.h> + +namespace searchcorespi::index { + +namespace { + +void expect_index_disk_dir(IndexDiskDir exp, const vespalib::string& dir) +{ + auto act = IndexDiskLayout::get_index_disk_dir(dir); + ASSERT_TRUE(act.valid()); + ASSERT_EQ(exp, act); +} + +void expect_bad_index_disk_dir(const vespalib::string& dir) +{ + auto act = IndexDiskLayout::get_index_disk_dir(dir); + ASSERT_FALSE(act.valid()); +} + +} + +TEST(IndexDiskLayoutTest, get_index_disk_dir_works) +{ + { + SCOPED_TRACE("index.fusion.1"); + expect_index_disk_dir(IndexDiskDir(1, true), "index.fusion.1"); + } + { + SCOPED_TRACE("index.flush.2"); + expect_index_disk_dir(IndexDiskDir(2, false), "index.flush.2"); + } + { + SCOPED_TRACE("index.flush.3"); + expect_index_disk_dir(IndexDiskDir(3, false), "index.flush.3"); + } + { + SCOPED_TRACE("foo/bar/index.flush.4"); + expect_index_disk_dir(IndexDiskDir(4, false), "foo/bar/index.flush.4"); + } + { + SCOPED_TRACE("index.flush."); + expect_bad_index_disk_dir("index.flush."); + } + { + SCOPED_TRACE("index.flush.0"); + expect_bad_index_disk_dir("index.flush.0"); + } + { + SCOPED_TRACE("asdf"); + expect_bad_index_disk_dir("asdf"); + } +} + +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt b/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt index c1e2b2f3dd1..3995eb836fd 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt +++ b/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt @@ -1,13 +1,14 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchcorespi_index OBJECT SOURCES - activediskindexes.cpp diskindexcleaner.cpp + disk_indexes.cpp disk_index_stats.cpp eventlogger.cpp fusionrunner.cpp iindexmanager.cpp iindexcollection.cpp + index_disk_dir_state.cpp index_manager_explorer.cpp index_manager_stats.cpp indexcollection.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp b/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp deleted file mode 100644 index c7891709801..00000000000 --- a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "activediskindexes.h" -#include <cassert> - -using vespalib::string; - -namespace searchcorespi::index { - -ActiveDiskIndexes::ActiveDiskIndexes() = default; -ActiveDiskIndexes::~ActiveDiskIndexes() = default; - -void ActiveDiskIndexes::setActive(const string &index) { - std::lock_guard lock(_lock); - _active.insert(index); -} - -void ActiveDiskIndexes::notActive(const string & index) { - std::lock_guard lock(_lock); - auto it = _active.find(index); - assert(it != _active.end()); - _active.erase(it); -} - -bool ActiveDiskIndexes::isActive(const string &index) const { - std::lock_guard lock(_lock); - return _active.find(index) != _active.end(); -} - -} diff --git a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.h b/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.h deleted file mode 100644 index a63dbb8b2a7..00000000000 --- a/searchcorespi/src/vespa/searchcorespi/index/activediskindexes.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <set> -#include <mutex> -#include <memory> - -namespace searchcorespi::index { - -/** - * Class used to keep track of the set of active disk indexes in an index maintainer. - * The index directories are used as identifiers. - */ -class ActiveDiskIndexes { - std::multiset<vespalib::string> _active; - mutable std::mutex _lock; - -public: - using SP = std::shared_ptr<ActiveDiskIndexes>; - ActiveDiskIndexes(); - ~ActiveDiskIndexes(); - ActiveDiskIndexes(const ActiveDiskIndexes &) = delete; - ActiveDiskIndexes & operator = (const ActiveDiskIndexes &) = delete; - void setActive(const vespalib::string & index); - void notActive(const vespalib::string & index); - bool isActive(const vespalib::string & index) const; -}; - -} diff --git a/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.cpp b/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.cpp new file mode 100644 index 00000000000..28f6a886d06 --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.cpp @@ -0,0 +1,131 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "disk_indexes.h" +#include "indexdisklayout.h" +#include "index_disk_dir.h" +#include "index_disk_dir_state.h" +#include <vespa/searchlib/util/dirtraverse.h> +#include <cassert> +#include <vector> + +using vespalib::string; + +namespace searchcorespi::index { + +DiskIndexes::DiskIndexes() = default; +DiskIndexes::~DiskIndexes() = default; + +void +DiskIndexes::setActive(const string &index, uint64_t size_on_disk) +{ + auto index_disk_dir = IndexDiskLayout::get_index_disk_dir(index); + assert(index_disk_dir.valid()); + std::lock_guard lock(_lock); + auto insres = _active.insert(std::make_pair(index_disk_dir, IndexDiskDirState())); + insres.first->second.activate(); + if (!insres.first->second.get_size_on_disk().has_value()) { + insres.first->second.set_size_on_disk(size_on_disk); + } +} + +void DiskIndexes::notActive(const string & index) { + auto index_disk_dir = IndexDiskLayout::get_index_disk_dir(index); + assert(index_disk_dir.valid()); + std::lock_guard lock(_lock); + auto it = _active.find(index_disk_dir); + assert(it != _active.end()); + assert(it->second.is_active()); + it->second.deactivate(); + if (!it->second.is_active()) { + _active.erase(it); + } +} + +bool DiskIndexes::isActive(const string &index) const { + auto index_disk_dir = IndexDiskLayout::get_index_disk_dir(index); + if (!index_disk_dir.valid()) { + return false; + } + std::lock_guard lock(_lock); + auto it = _active.find(index_disk_dir); + return (it != _active.end()) && it->second.is_active(); +} + + +void +DiskIndexes::add_not_active(IndexDiskDir index_disk_dir) +{ + std::lock_guard lock(_lock); + _active.insert(std::make_pair(index_disk_dir, IndexDiskDirState())); +} + +bool +DiskIndexes::remove(IndexDiskDir index_disk_dir) +{ + if (!index_disk_dir.valid()) { + return true; + } + std::lock_guard lock(_lock); + auto it = _active.find(index_disk_dir); + if (it == _active.end()) { + return true; + } + if (it->second.is_active()) { + return false; + } + _active.erase(it); + return true; +} + +uint64_t +DiskIndexes::get_transient_size(IndexDiskLayout& layout, IndexDiskDir index_disk_dir) const +{ + /* + * Only report transient size related to a valid fusion index. This ensures + * that transient size is reported once per index collection. + */ + if (!index_disk_dir.valid() || !index_disk_dir.is_fusion_index()) { + return 0u; + } + uint64_t transient_size = 0u; + std::vector<IndexDiskDir> deferred; + { + std::lock_guard lock(_lock); + for (auto &entry : _active) { + if (entry.first < index_disk_dir) { + /* + * Indexes before current fusion index are on the way out and + * will be removed when all older index collections + * referencing them are destroyed. Disk space used by these + * indexes is considered transient. + */ + if (entry.second.get_size_on_disk().has_value()) { + transient_size += entry.second.get_size_on_disk().value(); + } + } + if (index_disk_dir < entry.first && entry.first.is_fusion_index()) { + /* + * Fusion indexes after current fusion index can be partially + * complete and might be removed if fusion is aborted. Disk + * space used by these indexes is consider transient. + */ + if (entry.second.get_size_on_disk().has_value()) { + transient_size += entry.second.get_size_on_disk().value(); + } else { + deferred.emplace_back(entry.first); + } + } + } + } + for (auto& entry : deferred) { + auto index_dir = layout.getFusionDir(entry.get_id()); + try { + search::DirectoryTraverse dirt(index_dir.c_str()); + transient_size += dirt.GetTreeSize(); + } catch (std::exception &) { + } + } + return transient_size; +} + +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.h b/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.h new file mode 100644 index 00000000000..842c1814faf --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.h @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <map> +#include <mutex> +#include <memory> + +namespace searchcorespi::index { + +class IndexDiskDir; +class IndexDiskDirState; +class IndexDiskLayout; + +/** + * Class used to keep track of the set of disk indexes in an index maintainer. + * The index directories are used as identifiers. + * + * DiskIndexCleaner will remove old disk indexes not marked active, + * i.e. old disk indexes used by old index collections are not removed. + * + * At start of fusion, an entry for fusion output index is added, to allow for + * tracking of transient disk use while fusion is ongoing. If fusion fails then + * the entry is removed, otherwise the entry is marked active as a side effect + * of setting up a new index collection. + */ +class DiskIndexes { + std::map<IndexDiskDir, IndexDiskDirState> _active; + mutable std::mutex _lock; + +public: + using SP = std::shared_ptr<DiskIndexes>; + DiskIndexes(); + ~DiskIndexes(); + DiskIndexes(const DiskIndexes &) = delete; + DiskIndexes & operator = (const DiskIndexes &) = delete; + void setActive(const vespalib::string & index, uint64_t size_on_disk); + void notActive(const vespalib::string & index); + bool isActive(const vespalib::string & index) const; + void add_not_active(IndexDiskDir index_disk_dir); + bool remove(IndexDiskDir index_disk_dir); + uint64_t get_transient_size(IndexDiskLayout& layout, IndexDiskDir index_disk_dir) const; +}; + +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp index cce880eed3f..3bed7ea8ea7 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp @@ -1,7 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "diskindexcleaner.h" -#include "activediskindexes.h" +#include "disk_indexes.h" +#include "indexdisklayout.h" +#include "index_disk_dir.h" #include <vespa/fastos/file.h> #include <vespa/vespalib/io/fileutil.h> #include <sstream> @@ -80,13 +82,13 @@ bool isOldIndex(const string &index, uint32_t last_fusion_id) { } void removeOld(const string &base_dir, const vector<string> &indexes, - const ActiveDiskIndexes &active_indexes, bool remove) { + DiskIndexes &disk_indexes, bool remove) { uint32_t last_fusion_id = findLastFusionId(base_dir, indexes); for (size_t i = 0; i < indexes.size(); ++i) { const string index_dir = base_dir + "/" + indexes[i]; + auto index_disk_dir = IndexDiskLayout::get_index_disk_dir(indexes[i]); if (isOldIndex(indexes[i], last_fusion_id) && - !active_indexes.isActive(index_dir)) - { + disk_indexes.remove(index_disk_dir)) { if (remove) { removeDir(index_dir); } else { @@ -108,16 +110,16 @@ void removeInvalid(const string &base_dir, const vector<string> &indexes) { } // namespace void DiskIndexCleaner::clean(const string &base_dir, - const ActiveDiskIndexes &active_indexes) { + DiskIndexes &disk_indexes) { vector<string> indexes = readIndexes(base_dir); - removeOld(base_dir, indexes, active_indexes, false); + removeOld(base_dir, indexes, disk_indexes, false); removeInvalid(base_dir, indexes); } void DiskIndexCleaner::removeOldIndexes( - const string &base_dir, const ActiveDiskIndexes &active_indexes) { + const string &base_dir, DiskIndexes &disk_indexes) { vector<string> indexes = readIndexes(base_dir); - removeOld(base_dir, indexes, active_indexes, true); + removeOld(base_dir, indexes, disk_indexes, true); } } diff --git a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.h b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.h index 798193ab00b..cbd3a5aa94f 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.h +++ b/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.h @@ -6,7 +6,7 @@ namespace searchcorespi { namespace index { -class ActiveDiskIndexes; +class DiskIndexes; /** * Utility class used to clean and remove index directories. @@ -16,9 +16,9 @@ struct DiskIndexCleaner { * Deletes all indexes with id lower than the most recent fusion id. */ static void clean(const vespalib::string &index_dir, - const ActiveDiskIndexes& active_indexes); + DiskIndexes& disk_indexes); static void removeOldIndexes(const vespalib::string &index_dir, - const ActiveDiskIndexes& active_indexes); + DiskIndexes& disk_indexes); }; } // namespace index diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir.h b/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir.h new file mode 100644 index 00000000000..335838ddf2e --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir.h @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace searchcorespi::index { + +/* + * Class naming a disk index for a document type. + */ +class IndexDiskDir { + uint32_t _id; + bool _fusion; +public: + IndexDiskDir(uint32_t id, bool fusion) noexcept + : _id(id), + _fusion(fusion) + { + } + IndexDiskDir() noexcept + : IndexDiskDir(0, false) + { + } + bool operator<(const IndexDiskDir& rhs) const noexcept { + if (_id != rhs._id) { + return _id < rhs._id; + } + return !_fusion && rhs._fusion; + } + bool operator==(const IndexDiskDir& rhs) const noexcept { + return (_id == rhs._id) && (_fusion == rhs._fusion); + } + bool valid() const noexcept { return _id != 0u; } + bool is_fusion_index() const noexcept { return _fusion; } + uint64_t get_id() const noexcept { return _id; } +}; + +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.cpp b/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.cpp new file mode 100644 index 00000000000..ffe33d704c8 --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.cpp @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "index_disk_dir_state.h" +#include <cassert> + +namespace searchcorespi::index { + +void +IndexDiskDirState::deactivate() noexcept +{ + assert(_active_count > 0u); + --_active_count; +} + +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.h b/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.h new file mode 100644 index 00000000000..d8b790b3960 --- /dev/null +++ b/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> +#include <optional> + +namespace searchcorespi::index { + +/* + * Class describing state for a disk index directory. + */ +class IndexDiskDirState { + uint32_t _active_count; + std::optional<uint64_t> _size_on_disk; +public: + IndexDiskDirState() + : _active_count(0), + _size_on_disk() + { + } + + void activate() noexcept { ++_active_count; } + void deactivate() noexcept; + bool is_active() const noexcept { return _active_count != 0; } + const std::optional<uint64_t>& get_size_on_disk() const noexcept { return _size_on_disk; } + void set_size_on_disk(uint64_t size_on_disk) noexcept { _size_on_disk = size_on_disk; } +}; + +} diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.cpp index a13633751bb..c701d1dfb1d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "indexdisklayout.h" +#include "index_disk_dir.h" #include <sstream> namespace searchcorespi::index { @@ -53,4 +54,24 @@ IndexDiskLayout::getSelectorFileName(const vespalib::string &dir) return dir + "/selector"; } +IndexDiskDir +IndexDiskLayout::get_index_disk_dir(const vespalib::string& dir) +{ + auto name = dir.substr(dir.rfind('/') + 1); + const vespalib::string* prefix = nullptr; + bool fusion = false; + if (name.find(FlushDirPrefix) == 0) { + prefix = &FlushDirPrefix; + } else if (name.find(FusionDirPrefix) == 0) { + prefix = &FusionDirPrefix; + fusion = true; + } else { + return IndexDiskDir(); // invalid + } + std::istringstream ist(name.substr(prefix->size())); + uint32_t id = 0; + ist >> id; + return IndexDiskDir(id, fusion); // invalid if id == 0u +} + } diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.h b/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.h index 0598ad17c8a..94b35936cc7 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.h @@ -6,6 +6,8 @@ namespace searchcorespi { namespace index { +class IndexDiskDir; + /** * Utility class used to get static aspects of the disk layout (i.e directory and file names) * needed by the index maintainer. @@ -23,6 +25,7 @@ public: IndexDiskLayout(const vespalib::string &baseDir); vespalib::string getFlushDir(uint32_t sourceId) const; vespalib::string getFusionDir(uint32_t sourceId) const; + static IndexDiskDir get_index_disk_dir(const vespalib::string& dir); static vespalib::string getSerialNumFileName(const vespalib::string &dir); static vespalib::string getSchemaFileName(const vespalib::string &dir); diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index a2bd19c3d29..afe5b573f21 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -8,6 +8,7 @@ #include "indexmaintainer.h" #include "indexreadutilities.h" #include "indexwriteutilities.h" +#include "index_disk_dir.h" #include <vespa/searchcorespi/flush/lambdaflushtask.h> #include <vespa/searchlib/common/i_flush_token.h> #include <vespa/searchlib/index/schemautil.h> @@ -88,13 +89,22 @@ class DiskIndexWithDestructorCallback : public IDiskIndex { private: std::shared_ptr<IDestructorCallback> _callback; IDiskIndex::SP _index; + IndexDiskDir _index_disk_dir; + IndexDiskLayout& _layout; + DiskIndexes& _disk_indexes; public: DiskIndexWithDestructorCallback(IDiskIndex::SP index, - std::shared_ptr<IDestructorCallback> callback) noexcept + std::shared_ptr<IDestructorCallback> callback, + IndexDiskLayout& layout, + DiskIndexes& disk_indexes) noexcept : _callback(std::move(callback)), - _index(std::move(index)) - { } + _index(std::move(index)), + _index_disk_dir(IndexDiskLayout::get_index_disk_dir(_index->getIndexDir())), + _layout(layout), + _disk_indexes(disk_indexes) + { + } ~DiskIndexWithDestructorCallback() override; const IDiskIndex &getWrapped() const { return *_index; } @@ -117,7 +127,7 @@ public: { return _index->createBlueprint(requestContext, fields, term); } - search::SearchableStats getSearchableStats() const override { return _index->getSearchableStats(); } + search::SearchableStats getSearchableStats() const override; search::SerialNum getSerialNum() const override { return _index->getSerialNum(); } @@ -142,6 +152,16 @@ public: DiskIndexWithDestructorCallback::~DiskIndexWithDestructorCallback() = default; +search::SearchableStats +DiskIndexWithDestructorCallback::getSearchableStats() const +{ + auto stats = _index->getSearchableStats(); + uint64_t transient_size = _disk_indexes.get_transient_size(_layout, _index_disk_dir); + stats.fusion_size_on_disk(transient_size); + return stats; +} + + } // namespace IndexMaintainer::FusionArgs::FusionArgs() @@ -271,7 +291,7 @@ IndexMaintainer::updateActiveFusionPrunedSchema(const Schema &schema) void IndexMaintainer::deactivateDiskIndexes(vespalib::string indexDir) { - _active_indexes->notActive(indexDir); + _disk_indexes->notActive(indexDir); removeOldDiskIndexes(); } @@ -283,10 +303,13 @@ IndexMaintainer::loadDiskIndex(const string &indexDir) EventLogger::diskIndexLoadStart(indexDir); } vespalib::Timer timer; - _active_indexes->setActive(indexDir); + auto index = _operations.loadDiskIndex(indexDir); + auto stats = index->getSearchableStats(); + _disk_indexes->setActive(indexDir, stats.sizeOnDisk()); auto retval = std::make_shared<DiskIndexWithDestructorCallback>( - _operations.loadDiskIndex(indexDir), - makeLambdaCallback([this, indexDir]() { deactivateDiskIndexes(indexDir); })); + std::move(index), + makeLambdaCallback([this, indexDir]() { deactivateDiskIndexes(indexDir); }), + _layout, *_disk_indexes); if (LOG_WOULD_LOG(event)) { EventLogger::diskIndexLoadComplete(indexDir, vespalib::count_ms(timer.elapsed())); } @@ -302,11 +325,14 @@ IndexMaintainer::reloadDiskIndex(const IDiskIndex &oldIndex) EventLogger::diskIndexLoadStart(indexDir); } vespalib::Timer timer; - _active_indexes->setActive(indexDir); const IDiskIndex &wrappedDiskIndex = (dynamic_cast<const DiskIndexWithDestructorCallback &>(oldIndex)).getWrapped(); + auto index = _operations.reloadDiskIndex(wrappedDiskIndex); + auto stats = index->getSearchableStats(); + _disk_indexes->setActive(indexDir, stats.sizeOnDisk()); auto retval = std::make_shared<DiskIndexWithDestructorCallback>( - _operations.reloadDiskIndex(wrappedDiskIndex), - makeLambdaCallback([this, indexDir]() { deactivateDiskIndexes(indexDir); })); + std::move(index), + makeLambdaCallback([this, indexDir]() { deactivateDiskIndexes(indexDir); }), + _layout, *_disk_indexes); if (LOG_WOULD_LOG(event)) { EventLogger::diskIndexLoadComplete(indexDir, vespalib::count_ms(timer.elapsed())); } @@ -843,7 +869,7 @@ IndexMaintainer::IndexMaintainer(const IndexMaintainerConfig &config, IIndexMaintainerOperations &operations) : _base_dir(config.getBaseDir()), _warmupConfig(config.getWarmup()), - _active_indexes(std::make_shared<ActiveDiskIndexes>()), + _disk_indexes(std::make_shared<DiskIndexes>()), _layout(config.getBaseDir()), _schema(config.getSchema()), _activeFusionSchema(), @@ -876,7 +902,7 @@ IndexMaintainer::IndexMaintainer(const IndexMaintainerConfig &config, { // Called by document db init executor thread _changeGens.bumpPruneGen(); - DiskIndexCleaner::clean(_base_dir, *_active_indexes); + DiskIndexCleaner::clean(_base_dir, *_disk_indexes); FusionSpec spec = IndexReadUtilities::readFusionSpec(_base_dir); _next_id = 1 + (spec.flush_ids.empty() ? spec.last_fusion_id : spec.flush_ids.back()); _last_fusion_id = spec.last_fusion_id; @@ -1012,6 +1038,27 @@ IndexMaintainer::doFusion(SerialNum serialNum, std::shared_ptr<search::IFlushTok return getFusionDir(new_fusion_id); } +namespace { + +class RemoveFusionIndexGuard { + DiskIndexes* _disk_indexes; + IndexDiskDir _index_disk_dir; +public: + RemoveFusionIndexGuard(DiskIndexes& disk_indexes, IndexDiskDir index_disk_dir) + : _disk_indexes(&disk_indexes), + _index_disk_dir(index_disk_dir) + { + _disk_indexes->add_not_active(index_disk_dir); + } + ~RemoveFusionIndexGuard() { + if (_disk_indexes != nullptr) { + (void) _disk_indexes->remove(_index_disk_dir); + } + } + void reset() { _disk_indexes = nullptr; } +}; + +} uint32_t IndexMaintainer::runFusion(const FusionSpec &fusion_spec, std::shared_ptr<search::IFlushToken> flush_token) @@ -1033,6 +1080,8 @@ IndexMaintainer::runFusion(const FusionSpec &fusion_spec, std::shared_ptr<search if (FastOS_File::Stat(lastSerialFile.c_str(), &statInfo)) { serialNum = IndexReadUtilities::readSerialNum(lastFlushDir); } + IndexDiskDir fusion_index_disk_dir(fusion_spec.flush_ids.back(), true); + RemoveFusionIndexGuard remove_fusion_index_guard(*_disk_indexes, fusion_index_disk_dir); FusionRunner fusion_runner(_base_dir, args._schema, tuneFileAttributes, _ctx.getFileHeaderContext()); uint32_t new_fusion_id = fusion_runner.fuse(fusion_spec, serialNum, _operations, flush_token); bool ok = (new_fusion_id != 0); @@ -1065,6 +1114,7 @@ IndexMaintainer::runFusion(const FusionSpec &fusion_spec, std::shared_ptr<search } ChangeGens changeGens = getChangeGens(); IDiskIndex::SP new_index(loadDiskIndex(new_fusion_dir)); + remove_fusion_index_guard.reset(); // Post processing after fusion operation has completed and new disk // index has been opened. @@ -1100,7 +1150,7 @@ void IndexMaintainer::removeOldDiskIndexes() { LockGuard slock(_remove_lock); - DiskIndexCleaner::removeOldIndexes(_base_dir, *_active_indexes); + DiskIndexCleaner::removeOldIndexes(_base_dir, *_disk_indexes); } IndexMaintainer::FlushStats diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index 8213c02b90c..fa7a9145a3c 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -2,7 +2,7 @@ #pragma once #include "iindexmanager.h" -#include "activediskindexes.h" +#include "disk_indexes.h" #include "fusionspec.h" #include "idiskindex.h" #include "iindexmaintaineroperations.h" @@ -76,7 +76,7 @@ class IndexMaintainer : public IIndexManager, const vespalib::string _base_dir; const WarmupConfig _warmupConfig; - ActiveDiskIndexes::SP _active_indexes; + DiskIndexes::SP _disk_indexes; IndexDiskLayout _layout; Schema _schema; // Protected by SL + IUL Schema::SP _activeFusionSchema; // Protected by SL + IUL diff --git a/searchlib/src/tests/diskindex/fusion/.gitignore b/searchlib/src/tests/diskindex/fusion/.gitignore index d9a33665c43..934c9efb8fa 100644 --- a/searchlib/src/tests/diskindex/fusion/.gitignore +++ b/searchlib/src/tests/diskindex/fusion/.gitignore @@ -21,10 +21,7 @@ mdump2 mdump3 mdump4 mdump5 -sdump2 -sdump3 -sdump4 -sdump5 +sdump[2-6] /ddump6 /dmdump6 /dump6 diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp index 6794b9c0f5c..72867edf474 100644 --- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp +++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp @@ -24,7 +24,7 @@ #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> -#include <gtest/gtest.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("fusion_test"); @@ -65,7 +65,7 @@ protected: Schema _schema; const Schema & getSchema() const { return _schema; } - void requireThatFusionIsWorking(const vespalib::string &prefix, bool directio, bool readmmap); + void requireThatFusionIsWorking(const vespalib::string &prefix, bool directio, bool readmmap, bool force_short_merge_chunk); void make_simple_index(const vespalib::string &dump_dir, const IFieldLengthInspector &field_length_inspector); bool try_merge_simple_indexes(const vespalib::string &dump_dir, const std::vector<vespalib::string> &sources, std::shared_ptr<IFlushToken> flush_token); void merge_simple_indexes(const vespalib::string &dump_dir, const std::vector<vespalib::string> &sources); @@ -292,7 +292,7 @@ VESPA_THREAD_STACK_TAG(invert_executor) VESPA_THREAD_STACK_TAG(push_executor) void -FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool directio, bool readmmap) +FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool directio, bool readmmap, bool force_small_merge_chunk) { Schema schema; Schema schema2; @@ -357,7 +357,6 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire ib.setPrefix(dump2dir); uint32_t numDocs = 12 + 1; uint32_t numWords = fic.getNumUniqueWords(); - bool dynamicKPosOcc = false; MockFieldLengthInspector mock_field_length_inspector; TuneFileIndexing tuneFileIndexing; TuneFileSearch tuneFileSearch; @@ -392,9 +391,10 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire std::vector<vespalib::string> sources; SelectorArray selector(numDocs, 0); sources.push_back(prefix + "dump2"); - ASSERT_TRUE(Fusion::merge(schema, prefix + "dump3", sources, selector, - dynamicKPosOcc, - tuneFileIndexing,fileHeaderContext, executor, std::make_shared<FlushToken>())); + Fusion fusion(schema, prefix + "dump3", sources, selector, + tuneFileIndexing,fileHeaderContext); + fusion.set_force_small_merge_chunk(force_small_merge_chunk); + ASSERT_TRUE(fusion.merge(executor, std::make_shared<FlushToken>())); } while (0); do { DiskIndex dw3(prefix + "dump3"); @@ -405,9 +405,10 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire std::vector<vespalib::string> sources; SelectorArray selector(numDocs, 0); sources.push_back(prefix + "dump3"); - ASSERT_TRUE(Fusion::merge(schema2, prefix + "dump4", sources, selector, - dynamicKPosOcc, - tuneFileIndexing, fileHeaderContext, executor, std::make_shared<FlushToken>())); + Fusion fusion(schema2, prefix + "dump4", sources, selector, + tuneFileIndexing, fileHeaderContext); + fusion.set_force_small_merge_chunk(force_small_merge_chunk); + ASSERT_TRUE(fusion.merge(executor, std::make_shared<FlushToken>())); } while (0); do { DiskIndex dw4(prefix + "dump4"); @@ -418,9 +419,10 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire std::vector<vespalib::string> sources; SelectorArray selector(numDocs, 0); sources.push_back(prefix + "dump3"); - ASSERT_TRUE(Fusion::merge(schema3, prefix + "dump5", sources, selector, - dynamicKPosOcc, - tuneFileIndexing, fileHeaderContext, executor, std::make_shared<FlushToken>())); + Fusion fusion(schema3, prefix + "dump5", sources, selector, + tuneFileIndexing, fileHeaderContext); + fusion.set_force_small_merge_chunk(force_small_merge_chunk); + ASSERT_TRUE(fusion.merge(executor, std::make_shared<FlushToken>())); } while (0); do { DiskIndex dw5(prefix + "dump5"); @@ -431,9 +433,11 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire std::vector<vespalib::string> sources; SelectorArray selector(numDocs, 0); sources.push_back(prefix + "dump3"); - ASSERT_TRUE(Fusion::merge(schema, prefix + "dump6", sources, selector, - !dynamicKPosOcc, - tuneFileIndexing, fileHeaderContext, executor, std::make_shared<FlushToken>())); + Fusion fusion(schema, prefix + "dump6", sources, selector, + tuneFileIndexing, fileHeaderContext); + fusion.set_dynamic_k_pos_index_format(true); + fusion.set_force_small_merge_chunk(force_small_merge_chunk); + ASSERT_TRUE(fusion.merge(executor, std::make_shared<FlushToken>())); } while (0); do { DiskIndex dw6(prefix + "dump6"); @@ -444,9 +448,10 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire std::vector<vespalib::string> sources; SelectorArray selector(numDocs, 0); sources.push_back(prefix + "dump2"); - ASSERT_TRUE(Fusion::merge(schema, prefix + "dump3", sources, selector, - dynamicKPosOcc, - tuneFileIndexing, fileHeaderContext, executor, std::make_shared<FlushToken>())); + Fusion fusion(schema, prefix + "dump3", sources, selector, + tuneFileIndexing, fileHeaderContext); + fusion.set_force_small_merge_chunk(force_small_merge_chunk); + ASSERT_TRUE(fusion.merge(executor, std::make_shared<FlushToken>())); } while (0); do { DiskIndex dw3(prefix + "dump3"); @@ -487,9 +492,9 @@ FusionTest::try_merge_simple_indexes(const vespalib::string &dump_dir, const std TuneFileIndexing tuneFileIndexing; DummyFileHeaderContext fileHeaderContext; SelectorArray selector(20, 0); - return Fusion::merge(_schema, dump_dir, sources, selector, - false, - tuneFileIndexing, fileHeaderContext, executor, flush_token); + Fusion fusion(_schema, dump_dir, sources, selector, + tuneFileIndexing, fileHeaderContext); + return fusion.merge(executor, flush_token); } void @@ -506,22 +511,27 @@ FusionTest::FusionTest() TEST_F(FusionTest, require_that_normal_fusion_is_working) { - requireThatFusionIsWorking("", false, false); + requireThatFusionIsWorking("", false, false, false); } TEST_F(FusionTest, require_that_directio_fusion_is_working) { - requireThatFusionIsWorking("d", true, false); + requireThatFusionIsWorking("d", true, false, false); } TEST_F(FusionTest, require_that_mmap_fusion_is_working) { - requireThatFusionIsWorking("m", false, true); + requireThatFusionIsWorking("m", false, true, false); } TEST_F(FusionTest, require_that_directiommap_fusion_is_working) { - requireThatFusionIsWorking("dm", true, true); + requireThatFusionIsWorking("dm", true, true, false); +} + +TEST_F(FusionTest, require_that_small_merge_chunk_fusion_is_working) +{ + requireThatFusionIsWorking("s", false, false, true); } namespace { @@ -608,11 +618,11 @@ TEST_F(FusionTest, require_that_fusion_can_be_stopped) vespalib::rmdir("stopdump3", true); flush_token = std::make_shared<MyFlushToken>(1); ASSERT_FALSE(try_merge_simple_indexes("stopdump3", {"stopdump2"}, flush_token)); - EXPECT_EQ(12, flush_token->get_checks()); + EXPECT_EQ(8, flush_token->get_checks()); vespalib::rmdir("stopdump3", true); flush_token = std::make_shared<MyFlushToken>(47); ASSERT_FALSE(try_merge_simple_indexes("stopdump3", {"stopdump2"}, flush_token)); - EXPECT_LT(48, flush_token->get_checks()); + EXPECT_LE(48, flush_token->get_checks()); clean_stopped_fusion_testdirs(); } diff --git a/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp b/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp index b89324c71f5..ed857d5776b 100644 --- a/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp +++ b/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp @@ -7,43 +7,35 @@ LOG_SETUP("searchable_stats_test"); using namespace search; -TEST(SearchableStatsTest, merge_also_tracks_max_size_on_disk_for_component) +TEST(SearchableStatsTest, stats_can_be_merged) { SearchableStats stats; EXPECT_EQ(0u, stats.memoryUsage().allocatedBytes()); EXPECT_EQ(0u, stats.docsInMemory()); EXPECT_EQ(0u, stats.sizeOnDisk()); - EXPECT_EQ(0u, stats.max_component_size_on_disk()); + EXPECT_EQ(0u, stats.fusion_size_on_disk()); { SearchableStats rhs; EXPECT_EQ(&rhs.memoryUsage(vespalib::MemoryUsage(100,0,0,0)), &rhs); EXPECT_EQ(&rhs.docsInMemory(10), &rhs); EXPECT_EQ(&rhs.sizeOnDisk(1000), &rhs); - EXPECT_EQ(1000u, rhs.max_component_size_on_disk()); + EXPECT_EQ(&rhs.fusion_size_on_disk(500), &rhs); EXPECT_EQ(&stats.merge(rhs), &stats); } EXPECT_EQ(100u, stats.memoryUsage().allocatedBytes()); EXPECT_EQ(10u, stats.docsInMemory()); EXPECT_EQ(1000u, stats.sizeOnDisk()); - EXPECT_EQ(1000u, stats.max_component_size_on_disk()); + EXPECT_EQ(500u, stats.fusion_size_on_disk()); stats.merge(SearchableStats() .memoryUsage(vespalib::MemoryUsage(150,0,0,0)) .docsInMemory(15) - .sizeOnDisk(1500)); + .sizeOnDisk(1500) + .fusion_size_on_disk(800)); EXPECT_EQ(250u, stats.memoryUsage().allocatedBytes()); EXPECT_EQ(25u, stats.docsInMemory()); EXPECT_EQ(2500u, stats.sizeOnDisk()); - EXPECT_EQ(1500u, stats.max_component_size_on_disk()); - - stats.merge(SearchableStats() - .memoryUsage(vespalib::MemoryUsage(120,0,0,0)) - .docsInMemory(12) - .sizeOnDisk(1200)); - EXPECT_EQ(370u, stats.memoryUsage().allocatedBytes()); - EXPECT_EQ(37u, stats.docsInMemory()); - EXPECT_EQ(3700u, stats.sizeOnDisk()); - EXPECT_EQ(1500u, stats.max_component_size_on_disk()); + EXPECT_EQ(1300u, stats.fusion_size_on_disk()); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp b/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp index ee5d72f1daa..68672a0a930 100644 --- a/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/field_merger.cpp @@ -13,7 +13,7 @@ #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/searchlib/util/dirtraverse.h> -#include <vespa/searchlib/util/postingpriorityqueue.h> +#include <vespa/searchlib/util/posting_priority_queue_merger.hpp> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/exceptions.h> @@ -39,6 +39,11 @@ namespace search::diskindex { namespace { +constexpr uint32_t renumber_word_ids_heap_limit = 4; +constexpr uint32_t renumber_word_ids_merge_chunk = 1000000; +constexpr uint32_t merge_postings_heap_limit = 4; +constexpr uint32_t merge_postings_merge_chunk = 50000; + vespalib::string createTmpPath(const vespalib::string & base, uint32_t index) { vespalib::asciistream os; @@ -52,16 +57,20 @@ createTmpPath(const vespalib::string & base, uint32_t index) { FieldMerger::FieldMerger(uint32_t id, const FusionOutputIndex& fusion_out_index, std::shared_ptr<IFlushToken> flush_token) : _id(id), - _field_dir(fusion_out_index.get_path() + "/" + SchemaUtil::IndexIterator(fusion_out_index.get_schema(), id).getName()), + _field_name(SchemaUtil::IndexIterator(fusion_out_index.get_schema(), id).getName()), + _field_dir(fusion_out_index.get_path() + "/" + _field_name), _fusion_out_index(fusion_out_index), _flush_token(std::move(flush_token)), _word_readers(), _word_heap(), + _word_aggregator(), _word_num_mappings(), _num_word_ids(0), _readers(), _heap(), - _writer() + _writer(), + _state(State::MERGE_START), + _failed(false) { } @@ -107,14 +116,14 @@ bool FieldMerger::open_input_word_readers() { _word_readers.reserve(_fusion_out_index.get_old_indexes().size()); - _word_heap = std::make_unique<PostingPriorityQueue<DictionaryWordReader>>(); + _word_heap = std::make_unique<PostingPriorityQueueMerger<DictionaryWordReader, WordAggregator>>(); SchemaUtil::IndexIterator index(_fusion_out_index.get_schema(), _id); for (auto & oi : _fusion_out_index.get_old_indexes()) { auto reader(std::make_unique<DictionaryWordReader>()); const vespalib::string &tmpindexpath = createTmpPath(_field_dir, oi.getIndex()); const vespalib::string &oldindexpath = oi.getPath(); vespalib::string wordMapName = tmpindexpath + "/old2new.dat"; - vespalib::string fieldDir(oldindexpath + "/" + index.getName()); + vespalib::string fieldDir(oldindexpath + "/" + _field_name); vespalib::string dictName(fieldDir + "/dictionary"); const Schema &oldSchema = oi.getSchema(); if (!index.hasOldFields(oldSchema)) { @@ -163,24 +172,35 @@ FieldMerger::read_mapping_files() } bool -FieldMerger::renumber_word_ids() +FieldMerger::renumber_word_ids_start() { - SchemaUtil::IndexIterator index(_fusion_out_index.get_schema(), _id); - vespalib::string indexName = index.getName(); - LOG(debug, "Renumber word IDs for field %s", indexName.c_str()); - - WordAggregator out; - + LOG(debug, "Renumber word IDs for field %s", _field_name.c_str()); if (!open_input_word_readers()) { return false; } - _word_heap->merge(out, 4, *_flush_token); + _word_aggregator = std::make_unique<WordAggregator>(); + _word_heap->setup(renumber_word_ids_heap_limit); + _word_heap->set_merge_chunk(_fusion_out_index.get_force_small_merge_chunk() ? 1u : renumber_word_ids_merge_chunk); + return true; +} + +void +FieldMerger::renumber_word_ids_main() +{ + _word_heap->merge(*_word_aggregator, *_flush_token); if (_flush_token->stop_requested()) { - return false; + _failed = true; + } else if (_word_heap->empty()) { + _state = State::RENUMBER_WORD_IDS_FINISH; } - assert(_word_heap->empty()); +} + +bool +FieldMerger::renumber_word_ids_finish() +{ _word_heap.reset(); - _num_word_ids = out.getWordNum(); + _num_word_ids = _word_aggregator->getWordNum(); + _word_aggregator.reset(); // Close files for (auto &i : _word_readers) { @@ -193,11 +213,21 @@ FieldMerger::renumber_word_ids() if (!read_mapping_files()) { return false; } - LOG(debug, "Finished renumbering words IDs for field %s", indexName.c_str()); + LOG(debug, "Finished renumbering words IDs for field %s", _field_name.c_str()); return true; } +void +FieldMerger::renumber_word_ids_failed() +{ + _failed = true; + if (_flush_token->stop_requested()) { + return; + } + LOG(error, "Could not renumber field word ids for field %s dir %s", _field_name.c_str(), _field_dir.c_str()); +} + std::shared_ptr<FieldLengthScanner> FieldMerger::allocate_field_length_scanner() { @@ -226,7 +256,6 @@ FieldMerger::open_input_field_readers() _readers.reserve(_fusion_out_index.get_old_indexes().size()); SchemaUtil::IndexIterator index(_fusion_out_index.get_schema(), _id); auto field_length_scanner = allocate_field_length_scanner(); - vespalib::string indexName = index.getName(); for (const auto &oi : _fusion_out_index.get_old_indexes()) { const Schema &oldSchema = oi.getSchema(); if (!index.hasOldFields(oldSchema)) { @@ -234,7 +263,7 @@ FieldMerger::open_input_field_readers() } auto reader = FieldReader::allocFieldReader(index, oldSchema, field_length_scanner); reader->setup(_word_num_mappings[oi.getIndex()], oi.getDocIdMapping()); - if (!reader->open(oi.getPath() + "/" + indexName + "/", _fusion_out_index.get_tune_file_indexing()._read)) { + if (!reader->open(oi.getPath() + "/" + _field_name + "/", _fusion_out_index.get_tune_file_indexing()._read)) { return false; } _readers.push_back(std::move(reader)); @@ -322,7 +351,7 @@ FieldMerger::select_cooked_or_raw_features(FieldReader& reader) bool FieldMerger::setup_merge_heap() { - _heap = std::make_unique<PostingPriorityQueue<FieldReader>>(); + _heap = std::make_unique<PostingPriorityQueueMerger<FieldReader, FieldWriter>>(); for (auto &reader : _readers) { if (!select_cooked_or_raw_features(*reader)) { return false; @@ -334,16 +363,16 @@ FieldMerger::setup_merge_heap() _heap->initialAdd(reader.get()); } } + _heap->setup(merge_postings_heap_limit); + _heap->set_merge_chunk(_fusion_out_index.get_force_small_merge_chunk() ? 1u : merge_postings_merge_chunk); return true; } bool -FieldMerger::merge_postings() +FieldMerger::merge_postings_start() { - SchemaUtil::IndexIterator index(_fusion_out_index.get_schema(), _id); /* OUTPUT */ _writer = std::make_unique<FieldWriter>(_fusion_out_index.get_doc_id_limit(), _num_word_ids); - vespalib::string indexName = index.getName(); if (!open_input_field_readers()) { return false; @@ -351,15 +380,23 @@ FieldMerger::merge_postings() if (!open_field_writer()) { return false; } - if (!setup_merge_heap()) { - return false; - } + return setup_merge_heap(); +} - _heap->merge(*_writer, 4, *_flush_token); +void +FieldMerger::merge_postings_main() +{ + _heap->merge(*_writer, *_flush_token); if (_flush_token->stop_requested()) { - return false; + _failed = true; + } else if (_heap->empty()) { + _state = State::MERGE_POSTINGS_FINISH; } - assert(_heap->empty()); +} + +bool +FieldMerger::merge_postings_finish() +{ _heap.reset(); for (auto &reader : _readers) { @@ -375,55 +412,108 @@ FieldMerger::merge_postings() return true; } -bool -FieldMerger::merge_field() +void +FieldMerger::merge_postings_failed() +{ + _failed = true; + if (_flush_token->stop_requested()) { + return; + } + throw IllegalArgumentException(make_string("Could not merge field postings for field %s dir %s", + _field_name.c_str(), _field_dir.c_str())); +} + +void +FieldMerger::merge_field_start() { const Schema &schema = _fusion_out_index.get_schema(); SchemaUtil::IndexIterator index(schema, _id); - const vespalib::string &indexName = index.getName(); SchemaUtil::IndexSettings settings = index.getIndexSettings(); if (settings.hasError()) { - return false; + _failed = true; + return; } if (FileKit::hasStamp(_field_dir + "/.mergeocc_done")) { - return true; + _state = State::MERGE_DONE; + return; } vespalib::mkdir(_field_dir, false); - LOG(debug, "merge_field for field %s dir %s", indexName.c_str(), _field_dir.c_str()); + LOG(debug, "merge_field for field %s dir %s", _field_name.c_str(), _field_dir.c_str()); make_tmp_dirs(); - if (!renumber_word_ids()) { - if (_flush_token->stop_requested()) { - return false; - } - LOG(error, "Could not renumber field word ids for field %s dir %s", indexName.c_str(), _field_dir.c_str()); - return false; + if (!renumber_word_ids_start()) { + renumber_word_ids_failed(); + return; } + _state = State::RENUMBER_WORD_IDS; +} - // Tokamak - bool res = merge_postings(); +void +FieldMerger::merge_field_finish() +{ + bool res = merge_postings_finish(); if (!res) { - if (_flush_token->stop_requested()) { - return false; - } - throw IllegalArgumentException(make_string("Could not merge field postings for field %s dir %s", - indexName.c_str(), _field_dir.c_str())); + merge_postings_failed(); + _failed = true; + return; } if (!FileKit::createStamp(_field_dir + "/.mergeocc_done")) { - return false; + _failed = true; + return; } vespalib::File::sync(_field_dir); if (!clean_tmp_dirs()) { - return false; + _failed = true; + return; } - LOG(debug, "Finished merge_field for field %s dir %s", indexName.c_str(), _field_dir.c_str()); + LOG(debug, "Finished merge_field for field %s dir %s", _field_name.c_str(), _field_dir.c_str()); - return true; + _state = State::MERGE_DONE; +} + +void +FieldMerger::process_merge_field() +{ + switch (_state) { + case State::MERGE_START: + merge_field_start(); + break; + case State::RENUMBER_WORD_IDS: + renumber_word_ids_main(); + break; + case State::RENUMBER_WORD_IDS_FINISH: + if (!renumber_word_ids_finish()) { + renumber_word_ids_failed(); + } else if (!merge_postings_start()) { + merge_postings_failed(); + } else { + _state = State::MERGE_POSTINGS; + } + break; + case State::MERGE_POSTINGS: + merge_postings_main(); + break; + case State::MERGE_POSTINGS_FINISH: + merge_field_finish(); + break; + case State::MERGE_DONE: + default: + LOG_ABORT("should not be reached"); + } +} + +bool +FieldMerger::merge_field() +{ + while (!_failed && _state != State::MERGE_DONE) { + process_merge_field(); + } + return !_failed; } } diff --git a/searchlib/src/vespa/searchlib/diskindex/field_merger.h b/searchlib/src/vespa/searchlib/diskindex/field_merger.h index a57005e18a4..5017a7d5192 100644 --- a/searchlib/src/vespa/searchlib/diskindex/field_merger.h +++ b/searchlib/src/vespa/searchlib/diskindex/field_merger.h @@ -9,7 +9,7 @@ namespace search { class IFlushToken; -template <class IN> class PostingPriorityQueue; +template <class Reader, class Writer> class PostingPriorityQueueMerger; } namespace search::diskindex { @@ -19,6 +19,7 @@ class FieldLengthScanner; class FieldReader; class FieldWriter; class FusionOutputIndex; +class WordAggregator; class WordNumMapping; /* @@ -28,32 +29,55 @@ class FieldMerger { using WordNumMappingList = std::vector<WordNumMapping>; + enum class State { + MERGE_START, + RENUMBER_WORD_IDS, + RENUMBER_WORD_IDS_FINISH, + MERGE_POSTINGS, + MERGE_POSTINGS_FINISH, + MERGE_DONE + }; + uint32_t _id; + vespalib::string _field_name; vespalib::string _field_dir; const FusionOutputIndex& _fusion_out_index; std::shared_ptr<IFlushToken> _flush_token; std::vector<std::unique_ptr<DictionaryWordReader>> _word_readers; - std::unique_ptr<PostingPriorityQueue<DictionaryWordReader>> _word_heap; + std::unique_ptr<PostingPriorityQueueMerger<DictionaryWordReader, WordAggregator>> _word_heap; + std::unique_ptr<WordAggregator> _word_aggregator; WordNumMappingList _word_num_mappings; uint64_t _num_word_ids; std::vector<std::unique_ptr<FieldReader>> _readers; - std::unique_ptr<PostingPriorityQueue<FieldReader>> _heap; + std::unique_ptr<PostingPriorityQueueMerger<FieldReader, FieldWriter>> _heap; std::unique_ptr<FieldWriter> _writer; + State _state; + bool _failed; + bool _force_small_merge_chunk; void make_tmp_dirs(); bool clean_tmp_dirs(); bool open_input_word_readers(); bool read_mapping_files(); - bool renumber_word_ids(); + bool renumber_word_ids_start(); + void renumber_word_ids_main(); + bool renumber_word_ids_finish(); + void renumber_word_ids_failed(); std::shared_ptr<FieldLengthScanner> allocate_field_length_scanner(); bool open_input_field_readers(); bool open_field_writer(); bool select_cooked_or_raw_features(FieldReader& reader); bool setup_merge_heap(); - bool merge_postings(); + bool merge_postings_start(); + void merge_postings_main(); + bool merge_postings_finish(); + void merge_postings_failed(); public: FieldMerger(uint32_t id, const FusionOutputIndex& fusion_out_index, std::shared_ptr<IFlushToken> flush_token); ~FieldMerger(); + void merge_field_start(); + void merge_field_finish(); + void process_merge_field(); // Called multiple times bool merge_field(); }; diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index eafbbac361b..12552f09027 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -3,46 +3,29 @@ #include "fusion.h" #include "fusion_input_index.h" #include "field_merger.h" -#include "fieldreader.h" -#include "dictionarywordreader.h" -#include "field_length_scanner.h" -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/bitcompression/posocc_fields_params.h> +#include <vespa/fastos/file.h> +#include <vespa/searchlib/common/documentsummary.h> #include <vespa/searchlib/common/i_flush_token.h> -#include <vespa/searchlib/index/field_length_info.h> -#include <vespa/searchlib/util/filekit.h> +#include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/util/dirtraverse.h> -#include <vespa/searchlib/util/postingpriorityqueue.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/searchlib/common/documentsummary.h> +#include <vespa/vespalib/util/count_down_latch.h> #include <vespa/vespalib/util/error.h> +#include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/lambdatask.h> -#include <vespa/vespalib/util/count_down_latch.h> -#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/document/util/queue.h> -#include <sstream> #include <vespa/log/log.h> -#include <vespa/vespalib/util/exceptions.h> LOG_SETUP(".diskindex.fusion"); -using search::FileKit; -using search::PostingPriorityQueue; using search::common::FileHeaderContext; -using search::diskindex::DocIdMapping; -using search::diskindex::WordNumMapping; using search::docsummary::DocumentSummary; -using search::index::FieldLengthInfo; -using search::bitcompression::PosOccFieldParams; -using search::bitcompression::PosOccFieldsParams; -using search::index::PostingListParams; using search::index::Schema; using search::index::SchemaUtil; using search::index::schema::DataType; using vespalib::getLastErrorString; using vespalib::IllegalArgumentException; -using vespalib::make_string; namespace search::diskindex { @@ -51,6 +34,7 @@ namespace { std::vector<FusionInputIndex> createInputIndexes(const std::vector<vespalib::string> & sources, const SelectorArray &selector) { + assert(sources.size() <= 255); // due to source selector data type std::vector<FusionInputIndex> indexes; indexes.reserve(sources.size()); uint32_t i = 0; @@ -60,17 +44,28 @@ createInputIndexes(const std::vector<vespalib::string> & sources, const Selector return indexes; } +uint32_t calc_trimmed_doc_id_limit(const SelectorArray& selector, const std::vector<vespalib::string>& sources) +{ + uint32_t docIdLimit = selector.size(); + uint32_t trimmed_doc_id_limit = docIdLimit; + + // Limit docIdLimit in output based on selections that cannot be satisfied + uint32_t sources_size = sources.size(); + while (trimmed_doc_id_limit > 0 && selector[trimmed_doc_id_limit - 1] >= sources_size) { + --trimmed_doc_id_limit; + } + return trimmed_doc_id_limit; } -Fusion::Fusion(uint32_t docIdLimit, const Schema & schema, const vespalib::string & dir, - const std::vector<vespalib::string> & sources, const SelectorArray &selector, - bool dynamicKPosIndexFormat, const TuneFileIndexing &tuneFileIndexing, - const FileHeaderContext &fileHeaderContext) - : _fusion_out_index(schema, dir, createInputIndexes(sources, selector), docIdLimit, dynamicKPosIndexFormat, tuneFileIndexing, fileHeaderContext) +} + +Fusion::Fusion(const Schema& schema, const vespalib::string& dir, + const std::vector<vespalib::string>& sources, const SelectorArray& selector, + const TuneFileIndexing& tuneFileIndexing, + const FileHeaderContext& fileHeaderContext) + : _old_indexes(createInputIndexes(sources, selector)), + _fusion_out_index(schema, dir, _old_indexes, calc_trimmed_doc_id_limit(selector, sources), tuneFileIndexing, fileHeaderContext) { - if (!readSchemaFiles()) { - throw IllegalArgumentException("Cannot read schema files for source indexes"); - } } Fusion::~Fusion() = default; @@ -119,51 +114,41 @@ Fusion::readSchemaFiles() } bool -Fusion::merge(const Schema &schema, const vespalib::string &dir, const std::vector<vespalib::string> &sources, - const SelectorArray &selector, bool dynamicKPosOccFormat, - const TuneFileIndexing &tuneFileIndexing, const FileHeaderContext &fileHeaderContext, - vespalib::ThreadExecutor & executor, - std::shared_ptr<IFlushToken> flush_token) +Fusion::merge(vespalib::ThreadExecutor& executor, std::shared_ptr<IFlushToken> flush_token) { - assert(sources.size() <= 255); - uint32_t docIdLimit = selector.size(); - uint32_t trimmedDocIdLimit = docIdLimit; - - // Limit docIdLimit in output based on selections that cannot be satisfied - uint32_t sourcesSize = sources.size(); - while (trimmedDocIdLimit > 0 && selector[trimmedDocIdLimit - 1] >= sourcesSize) { - --trimmedDocIdLimit; - } - FastOS_StatInfo statInfo; - if (!FastOS_File::Stat(dir.c_str(), &statInfo)) { + if (!FastOS_File::Stat(_fusion_out_index.get_path().c_str(), &statInfo)) { if (statInfo._error != FastOS_StatInfo::FileNotFound) { - LOG(error, "Could not stat \"%s\"", dir.c_str()); + LOG(error, "Could not stat \"%s\"", _fusion_out_index.get_path().c_str()); return false; } } else { if (!statInfo._isDirectory) { - LOG(error, "\"%s\" is not a directory", dir.c_str()); + LOG(error, "\"%s\" is not a directory", _fusion_out_index.get_path().c_str()); return false; } - search::DirectoryTraverse dt(dir.c_str()); + search::DirectoryTraverse dt(_fusion_out_index.get_path().c_str()); if (!dt.RemoveTree()) { - LOG(error, "Failed to clean directory \"%s\"", dir.c_str()); + LOG(error, "Failed to clean directory \"%s\"", _fusion_out_index.get_path().c_str()); return false; } } - vespalib::mkdir(dir, false); - schema.saveToFile(dir + "/schema.txt"); - if (!DocumentSummary::writeDocIdLimit(dir, trimmedDocIdLimit)) { - LOG(error, "Could not write docsum count in dir %s: %s", dir.c_str(), getLastErrorString().c_str()); + vespalib::mkdir(_fusion_out_index.get_path(), false); + _fusion_out_index.get_schema().saveToFile(_fusion_out_index.get_path() + "/schema.txt"); + if (!DocumentSummary::writeDocIdLimit(_fusion_out_index.get_path(), _fusion_out_index.get_doc_id_limit())) { + LOG(error, "Could not write docsum count in dir %s: %s", _fusion_out_index.get_path().c_str(), getLastErrorString().c_str()); return false; } try { - auto fusion = std::make_unique<Fusion>(trimmedDocIdLimit, schema, dir, sources, selector, - dynamicKPosOccFormat, tuneFileIndexing, fileHeaderContext); - return fusion->mergeFields(executor, flush_token); + for (auto& old_index : _old_indexes) { + old_index.setup(); + } + if (!readSchemaFiles()) { + throw IllegalArgumentException("Cannot read schema files for source indexes"); + } + return mergeFields(executor, flush_token); } catch (const std::exception & e) { LOG(error, "%s", e.what()); return false; diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.h b/searchlib/src/vespa/searchlib/diskindex/fusion.h index 22dda4d6edf..1f5c4471950 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.h +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.h @@ -3,12 +3,10 @@ #pragma once #include "fusion_output_index.h" - #include <vespa/vespalib/util/threadexecutor.h> namespace search { class IFlushToken; -template <class IN> class PostingPriorityQueue; class TuneFileIndexing; } @@ -33,21 +31,19 @@ private: const Schema &getSchema() const { return _fusion_out_index.get_schema(); } + std::vector<FusionInputIndex> _old_indexes; FusionOutputIndex _fusion_out_index; public: Fusion(const Fusion &) = delete; Fusion& operator=(const Fusion &) = delete; - Fusion(uint32_t docIdLimit, const Schema &schema, const vespalib::string &dir, - const std::vector<vespalib::string> & sources, const SelectorArray &selector, bool dynamicKPosIndexFormat, - const TuneFileIndexing &tuneFileIndexing, const common::FileHeaderContext &fileHeaderContext); + Fusion(const Schema& schema, const vespalib::string& dir, + const std::vector<vespalib::string>& sources, const SelectorArray& selector, + const TuneFileIndexing& tuneFileIndexing, const common::FileHeaderContext& fileHeaderContext); ~Fusion(); - - static bool - merge(const Schema &schema, const vespalib::string &dir, const std::vector<vespalib::string> &sources, - const SelectorArray &docIdSelector, bool dynamicKPosOccFormat, const TuneFileIndexing &tuneFileIndexing, - const common::FileHeaderContext &fileHeaderContext, vespalib::ThreadExecutor & executor, - std::shared_ptr<IFlushToken> flush_token); + void set_dynamic_k_pos_index_format(bool dynamic_k_pos_index_format) { _fusion_out_index.set_dynamic_k_pos_index_format(dynamic_k_pos_index_format); } + void set_force_small_merge_chunk(bool force_small_merge_chunk) { _fusion_out_index.set_force_small_merge_chunk(force_small_merge_chunk); } + bool merge(vespalib::ThreadExecutor& executor, std::shared_ptr<IFlushToken> flush_token); }; } diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion_input_index.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion_input_index.cpp index 278d095b639..51c365957d9 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion_input_index.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion_input_index.cpp @@ -14,21 +14,28 @@ namespace search::diskindex { FusionInputIndex::FusionInputIndex(const vespalib::string& path, uint32_t index, const SelectorArray& selector) : _path(path), _index(index), - _schema() + _selector(&selector), + _schema(), + _docIdMapping() { - vespalib::string fname = path + "/schema.txt"; +} + +FusionInputIndex::~FusionInputIndex() = default; + +void +FusionInputIndex::setup() +{ + vespalib::string fname = _path + "/schema.txt"; if ( ! _schema.loadFromFile(fname)) { throw IllegalArgumentException(make_string("Failed loading schema %s", fname.c_str())); } if ( ! SchemaUtil::validateSchema(_schema)) { throw IllegalArgumentException(make_string("Failed validating schema %s", fname.c_str())); } - if (!_docIdMapping.readDocIdLimit(path)) { - throw IllegalArgumentException(make_string("Cannot determine docIdLimit for old index \"%s\"", path.c_str())); + if (!_docIdMapping.readDocIdLimit(_path)) { + throw IllegalArgumentException(make_string("Cannot determine docIdLimit for old index \"%s\"", _path.c_str())); } - _docIdMapping.setup(_docIdMapping._docIdLimit, &selector, index); + _docIdMapping.setup(_docIdMapping._docIdLimit, _selector, _index); } -FusionInputIndex::~FusionInputIndex() = default; - } diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion_input_index.h b/searchlib/src/vespa/searchlib/diskindex/fusion_input_index.h index fd7bb8f0256..6606e00d73b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion_input_index.h +++ b/searchlib/src/vespa/searchlib/diskindex/fusion_input_index.h @@ -14,10 +14,11 @@ namespace search::diskindex { class FusionInputIndex { private: - vespalib::string _path; - uint32_t _index; - index::Schema _schema; - DocIdMapping _docIdMapping; + vespalib::string _path; + uint32_t _index; + const SelectorArray* _selector; + index::Schema _schema; + DocIdMapping _docIdMapping; public: FusionInputIndex(const vespalib::string& path, uint32_t index, const SelectorArray& selector); @@ -25,6 +26,7 @@ public: FusionInputIndex & operator = (FusionInputIndex&&) = default; ~FusionInputIndex(); + void setup(); const vespalib::string& getPath() const noexcept { return _path; } uint32_t getIndex() const noexcept { return _index; } const DocIdMapping& getDocIdMapping() const noexcept { return _docIdMapping; } diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion_output_index.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion_output_index.cpp index 66ec0889cbe..3c75aa16b93 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion_output_index.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion_output_index.cpp @@ -5,12 +5,13 @@ namespace search::diskindex { -FusionOutputIndex::FusionOutputIndex(const index::Schema& schema, const vespalib::string& path, std::vector<FusionInputIndex> old_indexes, uint32_t doc_id_limit, bool dynamic_k_pos_index_format, const TuneFileIndexing& tune_file_indexing, const common::FileHeaderContext& file_header_context) +FusionOutputIndex::FusionOutputIndex(const index::Schema& schema, const vespalib::string& path, const std::vector<FusionInputIndex>& old_indexes, uint32_t doc_id_limit, const TuneFileIndexing& tune_file_indexing, const common::FileHeaderContext& file_header_context) : _schema(schema), _path(path), _old_indexes(std::move(old_indexes)), _doc_id_limit(doc_id_limit), - _dynamic_k_pos_index_format(dynamic_k_pos_index_format), + _dynamic_k_pos_index_format(false), + _force_small_merge_chunk(false), _tune_file_indexing(tune_file_indexing), _file_header_context(file_header_context) { diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion_output_index.h b/searchlib/src/vespa/searchlib/diskindex/fusion_output_index.h index c366b111363..729ecd26524 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion_output_index.h +++ b/searchlib/src/vespa/searchlib/diskindex/fusion_output_index.h @@ -22,20 +22,24 @@ class FusionOutputIndex private: const index::Schema& _schema; const vespalib::string _path; - const std::vector<FusionInputIndex> _old_indexes; + const std::vector<FusionInputIndex>& _old_indexes; const uint32_t _doc_id_limit; - const bool _dynamic_k_pos_index_format; + bool _dynamic_k_pos_index_format; + bool _force_small_merge_chunk; const TuneFileIndexing& _tune_file_indexing; const common::FileHeaderContext& _file_header_context; public: - FusionOutputIndex(const index::Schema& schema, const vespalib::string& path, std::vector<FusionInputIndex> old_indexes, uint32_t doc_id_limit, bool dynamic_k_pos_index_format, const TuneFileIndexing& tune_file_indexing, const common::FileHeaderContext& file_header_context); + FusionOutputIndex(const index::Schema& schema, const vespalib::string& path, const std::vector<FusionInputIndex>& old_indexes, uint32_t doc_id_limit, const TuneFileIndexing& tune_file_indexing, const common::FileHeaderContext& file_header_context); ~FusionOutputIndex(); + void set_dynamic_k_pos_index_format(bool dynamic_k_pos_index_format) { _dynamic_k_pos_index_format = dynamic_k_pos_index_format; } + void set_force_small_merge_chunk(bool force_small_merge_chunk) { _force_small_merge_chunk = force_small_merge_chunk; } const index::Schema& get_schema() const noexcept { return _schema; } const vespalib::string& get_path() const noexcept { return _path; } const std::vector<FusionInputIndex>& get_old_indexes() const noexcept { return _old_indexes; } uint32_t get_doc_id_limit() const noexcept { return _doc_id_limit; } bool get_dynamic_k_pos_index_format() const noexcept { return _dynamic_k_pos_index_format; } + bool get_force_small_merge_chunk() const noexcept { return _force_small_merge_chunk; } const TuneFileIndexing& get_tune_file_indexing() const noexcept { return _tune_file_indexing; } const common::FileHeaderContext& get_file_header_context() const noexcept { return _file_header_context; } }; diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp index 8ae3fe0bdad..666eed8f1e8 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/common/flush_token.h> #include <vespa/searchlib/memoryindex/posting_iterator.h> #include <vespa/searchlib/queryeval/iterators.h> -#include <vespa/searchlib/util/postingpriorityqueue.h> +#include <vespa/searchlib/util/posting_priority_queue_merger.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/btree/btreeiterator.hpp> #include <vespa/vespalib/btree/btreenode.hpp> @@ -352,7 +352,7 @@ FakeMemTreeOccFactory::setup(const std::vector<const FakeWord *> &fws) ++wordIdx; } - PostingPriorityQueue<FakeWord::RandomizedReader> heap; + PostingPriorityQueueMerger<FakeWord::RandomizedReader, FakeWord::RandomizedWriter> heap; std::vector<FakeWord::RandomizedReader>::iterator i(r.begin()); std::vector<FakeWord::RandomizedReader>::iterator ie(r.end()); FlushToken flush_token; @@ -363,8 +363,11 @@ FakeMemTreeOccFactory::setup(const std::vector<const FakeWord *> &fws) } ++i; } - heap.merge(_mgr, 4, flush_token); - assert(heap.empty()); + heap.setup(4); + heap.set_merge_chunk(100000); + while (!heap.empty()) { + heap.merge(_mgr, flush_token); + } _mgr.finalize(); } diff --git a/searchlib/src/vespa/searchlib/util/posting_priority_queue.h b/searchlib/src/vespa/searchlib/util/posting_priority_queue.h new file mode 100644 index 00000000000..c1549b32f93 --- /dev/null +++ b/searchlib/src/vespa/searchlib/util/posting_priority_queue.h @@ -0,0 +1,65 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vector> + +namespace search { + +/* + * Provide priority queue semantics for a set of posting readers. + */ +template <class Reader> +class PostingPriorityQueue +{ +protected: + class Ref + { + Reader *_ref; + public: + Ref(Reader *ref) + : _ref(ref) + { + } + + bool operator<(const Ref &rhs) const { return *_ref < *rhs._ref; } + Reader *get() const noexcept { return _ref; } + }; + + using Vector = std::vector<Ref>; + Vector _vec; + uint32_t _heap_limit; + uint32_t _merge_chunk; + +public: + PostingPriorityQueue() + : _vec(), + _heap_limit(0u), + _merge_chunk(0u) + { + } + + bool empty() const { return _vec.empty(); } + void clear() { _vec.clear(); } + void initialAdd(Reader *it) { _vec.push_back(Ref(it)); } + + /* + * Sort vector after a set of initial add operations, so lowest() + * and adjust() can be used. Skip sort if _vec.size() < heap_limit + * since merging with few elements don't use heap. + */ + void setup(uint32_t heap_limit); + + /* + * Return lowest value. Assumes vector is sorted. + */ + Reader *lowest() const { return _vec.front().get(); } + + /* + * The vector might no longer be sorted since the first element has changed + * value. Perform adjustments to make vector sorted again. + */ + void adjust(); +}; + +} diff --git a/searchlib/src/vespa/searchlib/util/posting_priority_queue.hpp b/searchlib/src/vespa/searchlib/util/posting_priority_queue.hpp new file mode 100644 index 00000000000..33f3bce2be6 --- /dev/null +++ b/searchlib/src/vespa/searchlib/util/posting_priority_queue.hpp @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "posting_priority_queue.h" + +namespace search { + +template <class Reader> +void +PostingPriorityQueue<Reader>::adjust() +{ + typedef typename Vector::iterator VIT; + if (!_vec.front().get()->isValid()) { + _vec.erase(_vec.begin()); // Iterator no longer valid + return; + } + if (_vec.size() == 1) { // Only one iterator left + return; + } + // Peform binary search to find first element higher than changed value + VIT gt = std::upper_bound(_vec.begin() + 1, _vec.end(), _vec.front()); + VIT to = _vec.begin(); + VIT from = to; + ++from; + Ref changed = *to; // Remember changed value + while (from != gt) { // Shift elements to make space for changed value + *to = *from; + ++from; + ++to; + } + *to = changed; // Save changed value at right location +} + +template <class Reader> +void +PostingPriorityQueue<Reader>::setup(uint32_t heap_limit) +{ + _heap_limit = heap_limit; + for (auto ref : _vec) { + assert(ref.get()->isValid()); + } + if (_vec.size() >= heap_limit) { + std::sort(_vec.begin(), _vec.end()); + } +} + +} diff --git a/searchlib/src/vespa/searchlib/util/posting_priority_queue_merger.h b/searchlib/src/vespa/searchlib/util/posting_priority_queue_merger.h new file mode 100644 index 00000000000..9debcd06ea6 --- /dev/null +++ b/searchlib/src/vespa/searchlib/util/posting_priority_queue_merger.h @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "posting_priority_queue.h" + +namespace search { + +/* + * Provide priority queue semantics for a set of posting readers with + * merging to a posting writer. + */ +template <class Reader, class Writer> +class PostingPriorityQueueMerger : public PostingPriorityQueue<Reader> +{ + uint32_t _merge_chunk; +public: + using Parent = PostingPriorityQueue<Reader>; + using Vector = typename Parent::Vector; + using Parent::_heap_limit; + using Parent::_vec; + using Parent::adjust; + using Parent::empty; + using Parent::lowest; + using Parent::setup; + + PostingPriorityQueueMerger() + : Parent(), + _merge_chunk(0u) + { + } + + void set_merge_chunk(uint32_t merge_chunk) { _merge_chunk = merge_chunk; } + void mergeHeap(Writer& writer, const IFlushToken& flush_token, uint32_t remaining_merge_chunk) __attribute__((noinline)); + static void mergeOne(Writer& writer, Reader& reader, const IFlushToken &flush_token, uint32_t remaining_merge_chunk) __attribute__((noinline)); + static void mergeTwo(Writer& writer, Reader& reader1, Reader& reader2, const IFlushToken& flush_token, uint32_t& remaining_merge_chunk) __attribute__((noinline)); + static void mergeSmall(Writer& writer, typename Vector::iterator ib, typename Vector::iterator ie, const IFlushToken &flush_token, uint32_t& remaining_merge_chunk) __attribute__((noinline)); + void merge(Writer& writer, const IFlushToken& flush_token) __attribute__((noinline)); +}; + +} diff --git a/searchlib/src/vespa/searchlib/util/posting_priority_queue_merger.hpp b/searchlib/src/vespa/searchlib/util/posting_priority_queue_merger.hpp new file mode 100644 index 00000000000..5676f6326df --- /dev/null +++ b/searchlib/src/vespa/searchlib/util/posting_priority_queue_merger.hpp @@ -0,0 +1,120 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "posting_priority_queue.hpp" +#include "posting_priority_queue_merger.h" + +namespace search { + +template <class Reader, class Writer> +void +PostingPriorityQueueMerger<Reader, Writer>::mergeHeap(Writer& writer, const IFlushToken& flush_token, uint32_t remaining_merge_chunk) +{ + while (remaining_merge_chunk > 0u && !empty() && !flush_token.stop_requested()) { + Reader *low = lowest(); + low->write(writer); + low->read(); + adjust(); + --remaining_merge_chunk; + } +} + +template <class Reader, class Writer> +void +PostingPriorityQueueMerger<Reader, Writer>::mergeOne(Writer& writer, Reader& reader, const IFlushToken& flush_token, uint32_t remaining_merge_chunk) +{ + while (remaining_merge_chunk > 0u && reader.isValid() && !flush_token.stop_requested()) { + reader.write(writer); + reader.read(); + --remaining_merge_chunk; + } +} + +template <class Reader, class Writer> +void +PostingPriorityQueueMerger<Reader, Writer>::mergeTwo(Writer& writer, Reader& reader1, Reader& reader2, const IFlushToken& flush_token, uint32_t& remaining_merge_chunk) +{ + while (remaining_merge_chunk > 0u && !flush_token.stop_requested()) { + Reader &low = reader2 < reader1 ? reader2 : reader1; + low.write(writer); + low.read(); + --remaining_merge_chunk; + if (!low.isValid()) { + break; + } + } +} + +template <class Reader, class Writer> +void +PostingPriorityQueueMerger<Reader, Writer>::mergeSmall(Writer& writer, typename Vector::iterator ib, typename Vector::iterator ie, const IFlushToken& flush_token, uint32_t& remaining_merge_chunk) +{ + while (remaining_merge_chunk > 0u && !flush_token.stop_requested()) { + typename Vector::iterator i = ib; + Reader *low = i->get(); + for (++i; i != ie; ++i) + if (*i->get() < *low) + low = i->get(); + low->write(writer); + low->read(); + --remaining_merge_chunk; + if (!low->isValid()) { + break; + } + } +} + +template <class Reader, class Writer> +void +PostingPriorityQueueMerger<Reader, Writer>::merge(Writer& writer, const IFlushToken& flush_token) +{ + if (_vec.empty()) + return; + assert(_heap_limit > 0u); + uint32_t remaining_merge_chunk = _merge_chunk; + if (_vec.size() >= _heap_limit) { + void (PostingPriorityQueueMerger::*mergeHeapFunc)(Writer& writer, const IFlushToken& flush_token, uint32_t remaining_merge_chunk) = + &PostingPriorityQueueMerger::mergeHeap; + (this->*mergeHeapFunc)(writer, flush_token, remaining_merge_chunk); + return; + } + while (remaining_merge_chunk > 0u && !flush_token.stop_requested()) { + if (_vec.size() == 1) { + void (*mergeOneFunc)(Writer& writer, Reader& reader, const IFlushToken& flush_token, uint32_t remaining_merge_chunk) = + &PostingPriorityQueueMerger::mergeOne; + (*mergeOneFunc)(writer, *_vec.front().get(), flush_token, remaining_merge_chunk); + if (!_vec.front().get()->isValid()) { + _vec.clear(); + } + return; + } + if (_vec.size() == 2) { + void (*mergeTwoFunc)(Writer& writer, Reader& reader1, Reader& reader2, const IFlushToken& flush_token, uint32_t& remaining_merge_chunk) = + &PostingPriorityQueueMerger::mergeTwo; + (*mergeTwoFunc)(writer, *_vec[0].get(), *_vec[1].get(), flush_token, remaining_merge_chunk); + } else { + void (*mergeSmallFunc)(Writer& writer, + typename Vector::iterator ib, + typename Vector::iterator ie, + const IFlushToken& flush_token, + uint32_t& remaining_merge_chunk) = + &PostingPriorityQueueMerger::mergeSmall; + (*mergeSmallFunc)(writer, _vec.begin(), _vec.end(), flush_token, remaining_merge_chunk); + } + for (typename Vector::iterator i = _vec.begin(), ie = _vec.end(); + i != ie; ++i) { + if (!i->get()->isValid()) { + _vec.erase(i); + break; + } + } + for (typename Vector::iterator i = _vec.begin(), ie = _vec.end(); + i != ie; ++i) { + assert(i->get()->isValid()); + } + assert(!_vec.empty()); + } +} + +} diff --git a/searchlib/src/vespa/searchlib/util/postingpriorityqueue.h b/searchlib/src/vespa/searchlib/util/postingpriorityqueue.h deleted file mode 100644 index c263c6bc470..00000000000 --- a/searchlib/src/vespa/searchlib/util/postingpriorityqueue.h +++ /dev/null @@ -1,261 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vector> - -namespace search -{ - -/* - * Provide priority queue semantics for a set of posting inputs. - */ -template <class IN> -class PostingPriorityQueue -{ -public: - class Ref - { - IN *_ref; - public: - Ref(IN *ref) - : _ref(ref) - { - } - - bool - operator<(const Ref &rhs) const - { - return *_ref < *rhs._ref; - } - - IN * - get() const - { - return _ref; - } - }; - - typedef std::vector<Ref> Vector; - Vector _vec; - - PostingPriorityQueue() - : _vec() - { - } - - bool - empty() const - { - return _vec.empty(); - } - - void - clear() - { - _vec.clear(); - } - - void - initialAdd(IN *it) - { - _vec.push_back(Ref(it)); - } - - /* - * Sort vector after a set of initial add operations, so lowest() - * and adjust() can be used. - */ - void - sort() - { - std::sort(_vec.begin(), _vec.end()); - } - - /* - * Return lowest value. Assumes vector is sorted. - */ - IN * - lowest() const - { - return _vec.front().get(); - } - - /* - * The vector might no longer be sorted since the first element has changed - * value. Perform adjustments to make vector sorted again. - */ - void - adjust(); - - - template <class OUT> - void - mergeHeap(OUT &out, const IFlushToken& flush_token) __attribute__((noinline)); - - template <class OUT> - static void - mergeOne(OUT &out, IN &in, const IFlushToken &flush_token) __attribute__((noinline)); - - template <class OUT> - static void - mergeTwo(OUT &out, IN &in1, IN &in2, const IFlushToken& flush_token) __attribute__((noinline)); - - template <class OUT> - static void - mergeSmall(OUT &out, - typename Vector::iterator ib, - typename Vector::iterator ie, - const IFlushToken &flush_token) - __attribute__((noinline)); - - template <class OUT> - void - merge(OUT &out, uint32_t heapLimit, const IFlushToken& flush_token) __attribute__((noinline)); -}; - - -template <class IN> -void -PostingPriorityQueue<IN>::adjust() -{ - typedef typename Vector::iterator VIT; - if (!_vec.front().get()->isValid()) { - _vec.erase(_vec.begin()); // Iterator no longer valid - return; - } - if (_vec.size() == 1) // Only one iterator left - return; - // Peform binary search to find first element higher than changed value - VIT gt = std::upper_bound(_vec.begin() + 1, _vec.end(), _vec.front()); - VIT to = _vec.begin(); - VIT from = to; - ++from; - Ref changed = *to; // Remember changed value - while (from != gt) { // Shift elements to make space for changed value - *to = *from; - ++from; - ++to; - } - *to = changed; // Save changed value at right location -} - - -template <class IN> -template <class OUT> -void -PostingPriorityQueue<IN>::mergeHeap(OUT &out, const IFlushToken& flush_token) -{ - while (!empty() && !flush_token.stop_requested()) { - IN *low = lowest(); - low->write(out); - low->read(); - adjust(); - } -} - - -template <class IN> -template <class OUT> -void -PostingPriorityQueue<IN>::mergeOne(OUT &out, IN &in, const IFlushToken& flush_token) -{ - while (in.isValid() && !flush_token.stop_requested()) { - in.write(out); - in.read(); - } -} - -template <class IN> -template <class OUT> -void -PostingPriorityQueue<IN>::mergeTwo(OUT &out, IN &in1, IN &in2, const IFlushToken& flush_token) -{ - while (!flush_token.stop_requested()) { - IN &low = in2 < in1 ? in2 : in1; - low.write(out); - low.read(); - if (!low.isValid()) - break; - } -} - - -template <class IN> -template <class OUT> -void -PostingPriorityQueue<IN>::mergeSmall(OUT &out, - typename Vector::iterator ib, - typename Vector::iterator ie, - const IFlushToken& flush_token) -{ - while (!flush_token.stop_requested()) { - typename Vector::iterator i = ib; - IN *low = i->get(); - for (++i; i != ie; ++i) - if (*i->get() < *low) - low = i->get(); - low->write(out); - low->read(); - if (!low->isValid()) - break; - } -} - - -template <class IN> -template <class OUT> -void -PostingPriorityQueue<IN>::merge(OUT &out, uint32_t heapLimit, const IFlushToken& flush_token) -{ - if (_vec.empty()) - return; - for (typename Vector::iterator i = _vec.begin(), ie = _vec.end(); i != ie; - ++i) { - assert(i->get()->isValid()); - } - if (_vec.size() >= heapLimit) { - sort(); - void (PostingPriorityQueue::*mergeHeapFunc)(OUT &out, const IFlushToken& flush_token) = - &PostingPriorityQueue::mergeHeap; - (this->*mergeHeapFunc)(out, flush_token); - return; - } - while (!flush_token.stop_requested()) { - if (_vec.size() == 1) { - void (*mergeOneFunc)(OUT &out, IN &in, const IFlushToken& flush_token) = - &PostingPriorityQueue<IN>::mergeOne; - (*mergeOneFunc)(out, *_vec.front().get(), flush_token); - _vec.clear(); - return; - } - if (_vec.size() == 2) { - void (*mergeTwoFunc)(OUT &out, IN &in1, IN &in2, const IFlushToken& flush_token) = - &PostingPriorityQueue<IN>::mergeTwo; - (*mergeTwoFunc)(out, *_vec[0].get(), *_vec[1].get(), flush_token); - } else { - void (*mergeSmallFunc)(OUT &out, - typename Vector::iterator ib, - typename Vector::iterator ie, - const IFlushToken& flush_token) = - &PostingPriorityQueue::mergeSmall; - (*mergeSmallFunc)(out, _vec.begin(), _vec.end(), flush_token); - } - for (typename Vector::iterator i = _vec.begin(), ie = _vec.end(); - i != ie; ++i) { - if (!i->get()->isValid()) { - _vec.erase(i); - break; - } - } - for (typename Vector::iterator i = _vec.begin(), ie = _vec.end(); - i != ie; ++i) { - assert(i->get()->isValid()); - } - assert(!_vec.empty()); - } -} - - -} // namespace search - diff --git a/searchlib/src/vespa/searchlib/util/searchable_stats.h b/searchlib/src/vespa/searchlib/util/searchable_stats.h index 970076c12d0..e785a4c4483 100644 --- a/searchlib/src/vespa/searchlib/util/searchable_stats.h +++ b/searchlib/src/vespa/searchlib/util/searchable_stats.h @@ -15,11 +15,11 @@ class SearchableStats private: vespalib::MemoryUsage _memoryUsage; size_t _docsInMemory; - size_t _sizeOnDisk; - size_t _max_component_size_on_disk; + size_t _sizeOnDisk; // in bytes + size_t _fusion_size_on_disk; // in bytes public: - SearchableStats() : _memoryUsage(), _docsInMemory(0), _sizeOnDisk(0), _max_component_size_on_disk(0) {} + SearchableStats() : _memoryUsage(), _docsInMemory(0), _sizeOnDisk(0), _fusion_size_on_disk(0) {} SearchableStats &memoryUsage(const vespalib::MemoryUsage &usage) { _memoryUsage = usage; return *this; @@ -32,22 +32,20 @@ public: size_t docsInMemory() const { return _docsInMemory; } SearchableStats &sizeOnDisk(size_t value) { _sizeOnDisk = value; - _max_component_size_on_disk = value; return *this; } size_t sizeOnDisk() const { return _sizeOnDisk; } - - /** - * Returns the max disk size used by a single Searchable component, - * e.g. among the components that are merged into a SearchableStats instance via merge(). - */ - size_t max_component_size_on_disk() const { return _max_component_size_on_disk; } + SearchableStats& fusion_size_on_disk(size_t value) { + _fusion_size_on_disk = value; + return *this; + } + size_t fusion_size_on_disk() const { return _fusion_size_on_disk; } SearchableStats &merge(const SearchableStats &rhs) { _memoryUsage.merge(rhs._memoryUsage); _docsInMemory += rhs._docsInMemory; _sizeOnDisk += rhs._sizeOnDisk; - _max_component_size_on_disk = std::max(_max_component_size_on_disk, rhs._sizeOnDisk); + _fusion_size_on_disk += rhs._fusion_size_on_disk; return *this; } }; diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java index 5ffb37feb70..d1a56b8ac8a 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/CloudConfigInstallVariables.java @@ -121,6 +121,12 @@ public class CloudConfigInstallVariables implements CloudConfigOptions { return getInstallVariable("zts_url"); } + @Override + public String zooKeeperSnapshotMethod() { + String vespaZookeeperSnapshotMethod = System.getenv("VESPA_ZOOKEEPER_SNAPSHOT_METHOD"); + return vespaZookeeperSnapshotMethod == null ? "" : vespaZookeeperSnapshotMethod; + } + static ConfigServer[] toConfigServers(String configserversString) { return multiValueParameterStream(configserversString) .map(CloudConfigInstallVariables::toConfigServer) diff --git a/storage/src/tests/persistence/CMakeLists.txt b/storage/src/tests/persistence/CMakeLists.txt index 7b165e11b66..fb8120210c1 100644 --- a/storage/src/tests/persistence/CMakeLists.txt +++ b/storage/src/tests/persistence/CMakeLists.txt @@ -12,6 +12,7 @@ vespa_add_executable(storage_persistence_gtest_runner_app TEST persistencethread_splittest.cpp processalltest.cpp provider_error_wrapper_test.cpp + shared_operation_throttler_test.cpp splitbitdetectortest.cpp testandsettest.cpp gtest_runner.cpp diff --git a/storage/src/tests/persistence/active_operations_stats_test.cpp b/storage/src/tests/persistence/active_operations_stats_test.cpp index a5dd3d929db..8caa84977ce 100644 --- a/storage/src/tests/persistence/active_operations_stats_test.cpp +++ b/storage/src/tests/persistence/active_operations_stats_test.cpp @@ -96,17 +96,17 @@ ActiveOperationsStatsTest::test_active_operations_stats() auto lock0 = filestorHandler->getNextMessage(stripeId); auto lock1 = filestorHandler->getNextMessage(stripeId); auto lock2 = filestorHandler->getNextMessage(stripeId); - ASSERT_TRUE(lock0.first); - ASSERT_TRUE(lock1.first); - ASSERT_FALSE(lock2.first); + ASSERT_TRUE(lock0.lock); + ASSERT_TRUE(lock1.lock); + ASSERT_FALSE(lock2.lock); auto stats = filestorHandler->get_active_operations_stats(false); { SCOPED_TRACE("during"); assert_active_operations_stats(stats, 2, 2, 0); } EXPECT_EQ(3, stats.get_total_size()); - lock0.first.reset(); - lock1.first.reset(); + lock0.lock.reset(); + lock1.lock.reset(); stats = filestorHandler->get_active_operations_stats(false); { SCOPED_TRACE("after"); diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index cd496605a6c..939e7ae7b6a 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -516,11 +516,11 @@ TEST_F(FileStorManagerTest, handler_priority) { filestorHandler.schedule(cmd); } - ASSERT_EQ(15, filestorHandler.getNextMessage(stripeId).second->getPriority()); - ASSERT_EQ(30, filestorHandler.getNextMessage(stripeId).second->getPriority()); - ASSERT_EQ(45, filestorHandler.getNextMessage(stripeId).second->getPriority()); - ASSERT_EQ(60, filestorHandler.getNextMessage(stripeId).second->getPriority()); - ASSERT_EQ(75, filestorHandler.getNextMessage(stripeId).second->getPriority()); + ASSERT_EQ(15, filestorHandler.getNextMessage(stripeId).msg->getPriority()); + ASSERT_EQ(30, filestorHandler.getNextMessage(stripeId).msg->getPriority()); + ASSERT_EQ(45, filestorHandler.getNextMessage(stripeId).msg->getPriority()); + ASSERT_EQ(60, filestorHandler.getNextMessage(stripeId).msg->getPriority()); + ASSERT_EQ(75, filestorHandler.getNextMessage(stripeId).msg->getPriority()); } class MessagePusherThread : public document::Runnable { @@ -570,7 +570,7 @@ public: void run() override { while (!_done) { FileStorHandler::LockedMessage msg = _handler.getNextMessage(_threadId); - if (msg.second.get()) { + if (msg.msg.get()) { uint32_t originalConfig = _config.load(); _fetchedCount++; std::this_thread::sleep_for(5ms); @@ -641,15 +641,15 @@ TEST_F(FileStorManagerTest, handler_pause) { filestorHandler.schedule(cmd); } - ASSERT_EQ(15, filestorHandler.getNextMessage(stripeId).second->getPriority()); + ASSERT_EQ(15, filestorHandler.getNextMessage(stripeId).msg->getPriority()); { ResumeGuard guard = filestorHandler.pause(); (void)guard; - ASSERT_EQ(filestorHandler.getNextMessage(stripeId).second.get(), nullptr); + ASSERT_EQ(filestorHandler.getNextMessage(stripeId).msg.get(), nullptr); } - ASSERT_EQ(30, filestorHandler.getNextMessage(stripeId).second->getPriority()); + ASSERT_EQ(30, filestorHandler.getNextMessage(stripeId).msg->getPriority()); } TEST_F(FileStorManagerTest, remap_split) { @@ -729,8 +729,8 @@ TEST_F(FileStorManagerTest, handler_timeout) { std::this_thread::sleep_for(51ms); for (;;) { auto lock = filestorHandler.getNextMessage(stripeId); - if (lock.first.get()) { - ASSERT_EQ(200, lock.second->getPriority()); + if (lock.lock.get()) { + ASSERT_EQ(200, lock.msg->getPriority()); break; } } @@ -2013,7 +2013,7 @@ expect_async_message(StorageMessage::Priority exp_pri, { EXPECT_TRUE(result.was_scheduled()); ASSERT_TRUE(result.has_async_message()); - EXPECT_EQ(exp_pri, result.async_message().second->getPriority()); + EXPECT_EQ(exp_pri, result.async_message().msg->getPriority()); } void @@ -2045,8 +2045,8 @@ TEST_F(FileStorHandlerTest, async_message_with_lowest_pri_returned_on_schedule) auto result = handler->schedule_and_get_next_async_message(make_put_command(30)); expect_async_message(20, result); } - EXPECT_EQ(30, get_next_message().second->getPriority()); - EXPECT_EQ(40, get_next_message().second->getPriority()); + EXPECT_EQ(30, get_next_message().msg->getPriority()); + EXPECT_EQ(40, get_next_message().msg->getPriority()); } TEST_F(FileStorHandlerTest, no_async_message_returned_if_lowest_pri_message_is_not_async) @@ -2057,8 +2057,8 @@ TEST_F(FileStorHandlerTest, no_async_message_returned_if_lowest_pri_message_is_n auto result = handler->schedule_and_get_next_async_message(make_put_command(30)); expect_empty_async_message(result); - EXPECT_EQ(20, get_next_message().second->getPriority()); - EXPECT_EQ(30, get_next_message().second->getPriority()); + EXPECT_EQ(20, get_next_message().msg->getPriority()); + EXPECT_EQ(30, get_next_message().msg->getPriority()); } TEST_F(FileStorHandlerTest, inhibited_operations_are_skipped) @@ -2079,7 +2079,7 @@ TEST_F(FileStorHandlerTest, inhibited_operations_are_skipped) expect_async_message(40, result); } } - EXPECT_EQ(30, get_next_message().second->getPriority()); + EXPECT_EQ(30, get_next_message().msg->getPriority()); } } // storage diff --git a/storage/src/tests/persistence/persistencequeuetest.cpp b/storage/src/tests/persistence/persistencequeuetest.cpp index 2557fa537f5..5f3836727dd 100644 --- a/storage/src/tests/persistence/persistencequeuetest.cpp +++ b/storage/src/tests/persistence/persistencequeuetest.cpp @@ -91,14 +91,14 @@ TEST_F(PersistenceQueueTest, fetch_next_unlocked_message_if_bucket_locked) { f.filestorHandler->schedule(createPut(5432, 0)); auto lock0 = f.filestorHandler->getNextMessage(f.stripeId); - ASSERT_TRUE(lock0.first.get()); + ASSERT_TRUE(lock0.lock.get()); EXPECT_EQ(document::BucketId(16, 1234), - dynamic_cast<api::PutCommand&>(*lock0.second).getBucketId()); + dynamic_cast<api::PutCommand&>(*lock0.msg).getBucketId()); auto lock1 = f.filestorHandler->getNextMessage(f.stripeId); - ASSERT_TRUE(lock1.first.get()); + ASSERT_TRUE(lock1.lock.get()); EXPECT_EQ(document::BucketId(16, 5432), - dynamic_cast<api::PutCommand&>(*lock1.second).getBucketId()); + dynamic_cast<api::PutCommand&>(*lock1.msg).getBucketId()); } TEST_F(PersistenceQueueTest, shared_locked_operations_allow_concurrent_bucket_access) { @@ -108,14 +108,14 @@ TEST_F(PersistenceQueueTest, shared_locked_operations_allow_concurrent_bucket_ac f.filestorHandler->schedule(createGet(1234)); auto lock0 = f.filestorHandler->getNextMessage(f.stripeId); - ASSERT_TRUE(lock0.first.get()); - EXPECT_EQ(api::LockingRequirements::Shared, lock0.first->lockingRequirements()); + ASSERT_TRUE(lock0.lock.get()); + EXPECT_EQ(api::LockingRequirements::Shared, lock0.lock->lockingRequirements()); // Even though we already have a lock on the bucket, Gets allow shared locking and we // should therefore be able to get another lock. auto lock1 = f.filestorHandler->getNextMessage(f.stripeId); - ASSERT_TRUE(lock1.first.get()); - EXPECT_EQ(api::LockingRequirements::Shared, lock1.first->lockingRequirements()); + ASSERT_TRUE(lock1.lock.get()); + EXPECT_EQ(api::LockingRequirements::Shared, lock1.lock->lockingRequirements()); } TEST_F(PersistenceQueueTest, exclusive_locked_operation_not_started_if_shared_op_active) { @@ -125,12 +125,12 @@ TEST_F(PersistenceQueueTest, exclusive_locked_operation_not_started_if_shared_op f.filestorHandler->schedule(createPut(1234, 0)); auto lock0 = f.filestorHandler->getNextMessage(f.stripeId); - ASSERT_TRUE(lock0.first.get()); - EXPECT_EQ(api::LockingRequirements::Shared, lock0.first->lockingRequirements()); + ASSERT_TRUE(lock0.lock.get()); + EXPECT_EQ(api::LockingRequirements::Shared, lock0.lock->lockingRequirements()); // Expected to time out auto lock1 = f.filestorHandler->getNextMessage(f.stripeId); - ASSERT_FALSE(lock1.first.get()); + ASSERT_FALSE(lock1.lock.get()); } TEST_F(PersistenceQueueTest, shared_locked_operation_not_started_if_exclusive_op_active) { @@ -140,12 +140,12 @@ TEST_F(PersistenceQueueTest, shared_locked_operation_not_started_if_exclusive_op f.filestorHandler->schedule(createGet(1234)); auto lock0 = f.filestorHandler->getNextMessage(f.stripeId); - ASSERT_TRUE(lock0.first.get()); - EXPECT_EQ(api::LockingRequirements::Exclusive, lock0.first->lockingRequirements()); + ASSERT_TRUE(lock0.lock.get()); + EXPECT_EQ(api::LockingRequirements::Exclusive, lock0.lock->lockingRequirements()); // Expected to time out auto lock1 = f.filestorHandler->getNextMessage(f.stripeId); - ASSERT_FALSE(lock1.first.get()); + ASSERT_FALSE(lock1.lock.get()); } TEST_F(PersistenceQueueTest, exclusive_locked_operation_not_started_if_exclusive_op_active) { @@ -155,12 +155,12 @@ TEST_F(PersistenceQueueTest, exclusive_locked_operation_not_started_if_exclusive f.filestorHandler->schedule(createPut(1234, 0)); auto lock0 = f.filestorHandler->getNextMessage(f.stripeId); - ASSERT_TRUE(lock0.first.get()); - EXPECT_EQ(api::LockingRequirements::Exclusive, lock0.first->lockingRequirements()); + ASSERT_TRUE(lock0.lock.get()); + EXPECT_EQ(api::LockingRequirements::Exclusive, lock0.lock->lockingRequirements()); // Expected to time out auto lock1 = f.filestorHandler->getNextMessage(f.stripeId); - ASSERT_FALSE(lock1.first.get()); + ASSERT_FALSE(lock1.lock.get()); } } // namespace storage diff --git a/storage/src/tests/persistence/shared_operation_throttler_test.cpp b/storage/src/tests/persistence/shared_operation_throttler_test.cpp new file mode 100644 index 00000000000..0ad380937c7 --- /dev/null +++ b/storage/src/tests/persistence/shared_operation_throttler_test.cpp @@ -0,0 +1,116 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/storage/persistence/shared_operation_throttler.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/barrier.h> +#include <chrono> +#include <thread> + +using namespace ::testing; + +namespace storage { + +using ThrottleToken = SharedOperationThrottler::Token; + +TEST(SharedOperationThrottlerTest, unlimited_throttler_does_not_throttle) { + // We technically can't test that the unlimited throttler _never_ throttles, but at + // least check that it doesn't throttle _twice_, and then induce from this ;) + auto throttler = SharedOperationThrottler::make_unlimited_throttler(); + auto token1 = throttler->try_acquire_one(); + EXPECT_TRUE(token1.valid()); + auto token2 = throttler->blocking_acquire_one(); + EXPECT_TRUE(token2.valid()); + // Window size should be zero (i.e. unlimited) for unlimited throttler + EXPECT_EQ(throttler->current_window_size(), 0); +} + +TEST(SharedOperationThrottlerTest, dynamic_throttler_respects_initial_window_size) { + auto throttler = SharedOperationThrottler::make_dynamic_throttler(1); + auto token1 = throttler->try_acquire_one(); + EXPECT_TRUE(token1.valid()); + auto token2 = throttler->try_acquire_one(); + EXPECT_FALSE(token2.valid()); + + EXPECT_EQ(throttler->current_window_size(), 1); +} + +TEST(SharedOperationThrottlerTest, blocking_acquire_returns_immediately_if_slot_available) { + auto throttler = SharedOperationThrottler::make_dynamic_throttler(1); + auto token = throttler->blocking_acquire_one(); + EXPECT_TRUE(token.valid()); + token.reset(); + token = throttler->blocking_acquire_one(600s); // Should never block. + EXPECT_TRUE(token.valid()); +} + +TEST(SharedOperationThrottlerTest, blocking_call_woken_up_if_throttle_slot_available) { + auto throttler = SharedOperationThrottler::make_dynamic_throttler(1); + vespalib::Barrier barrier(2); + std::thread t([&] { + auto token = throttler->try_acquire_one(); + assert(token.valid()); + barrier.await(); + while (throttler->waiting_threads() != 1) { + std::this_thread::sleep_for(100us); + } + // Implicit token release at thread scope exit + }); + barrier.await(); + auto token = throttler->blocking_acquire_one(); + EXPECT_TRUE(token.valid()); + t.join(); +} + +TEST(SharedOperationThrottlerTest, time_bounded_blocking_acquire_waits_for_timeout) { + auto throttler = SharedOperationThrottler::make_dynamic_throttler(1); + auto window_filling_token = throttler->try_acquire_one(); + auto before = std::chrono::steady_clock::now(); + // Will block for at least 1ms. Since no window slot will be available by that time, + // an invalid token should be returned. + auto token = throttler->blocking_acquire_one(1ms); + auto after = std::chrono::steady_clock::now(); + EXPECT_TRUE((after - before) >= 1ms); + EXPECT_FALSE(token.valid()); +} + +TEST(SharedOperationThrottlerTest, default_constructed_token_is_invalid) { + ThrottleToken token; + EXPECT_FALSE(token.valid()); + token.reset(); // no-op + EXPECT_FALSE(token.valid()); +} + +TEST(SharedOperationThrottlerTest, token_destruction_frees_up_throttle_window_slot) { + auto throttler = SharedOperationThrottler::make_dynamic_throttler(1); + { + auto token = throttler->try_acquire_one(); + EXPECT_TRUE(token.valid()); + } + auto token = throttler->try_acquire_one(); + EXPECT_TRUE(token.valid()); +} + +TEST(SharedOperationThrottlerTest, token_can_be_moved_and_reset) { + auto throttler = SharedOperationThrottler::make_dynamic_throttler(1); + auto token1 = throttler->try_acquire_one(); + auto token2 = std::move(token1); // move ctor + EXPECT_TRUE(token2.valid()); + EXPECT_FALSE(token1.valid()); + ThrottleToken token3; + token3 = std::move(token2); // move assignment op + EXPECT_TRUE(token3.valid()); + EXPECT_FALSE(token2.valid()); + + // Trying to fetch new token should not succeed due to active token and win size of 1 + token1 = throttler->try_acquire_one(); + EXPECT_FALSE(token1.valid()); + // Resetting the token should free up the slot in the window + token3.reset(); + token1 = throttler->try_acquire_one(); + EXPECT_TRUE(token1.valid()); +} + +// TODO ideally we'd test that the dynamic throttler has a window size that is actually +// dynamic, but the backing DynamicThrottlePolicy implementation is a black box so +// it's not trivial to know how to do this reliably. + +} diff --git a/storage/src/tests/persistence/splitbitdetectortest.cpp b/storage/src/tests/persistence/splitbitdetectortest.cpp index d4e84836a5a..5c4dc85e825 100644 --- a/storage/src/tests/persistence/splitbitdetectortest.cpp +++ b/storage/src/tests/persistence/splitbitdetectortest.cpp @@ -15,6 +15,8 @@ using namespace ::testing; namespace storage { +using DocEntryList = std::vector<spi::DocEntry::UP>; + struct SplitBitDetectorTest : Test { document::TestDocMan testDocMan; spi::dummy::DummyPersistence provider; @@ -33,7 +35,7 @@ struct SplitBitDetectorTest : Test { }; TEST_F(SplitBitDetectorTest, two_users) { - std::vector<spi::DocEntry::UP> entries; + DocEntryList entries; for (uint32_t i = 0; i < 5; ++i) { document::Document::SP doc( testDocMan.createRandomDocumentAtLocation(1, i, 1, 1)); @@ -54,7 +56,7 @@ TEST_F(SplitBitDetectorTest, two_users) { } TEST_F(SplitBitDetectorTest, single_user) { - std::vector<spi::DocEntry::UP> entries; + DocEntryList entries; for (uint32_t i = 0; i < 10; ++i) { document::Document::SP doc( testDocMan.createRandomDocumentAtLocation(1, i, 1, 1)); @@ -71,7 +73,7 @@ TEST_F(SplitBitDetectorTest, single_user) { TEST_F(SplitBitDetectorTest, max_bits) { int minContentSize = 1, maxContentSize = 1; - std::vector<spi::DocEntry::UP> entries; + DocEntryList entries; for (uint32_t seed = 0; seed < 10; ++seed) { int location = 1; document::Document::SP doc(testDocMan.createRandomDocumentAtLocation( @@ -92,7 +94,7 @@ TEST_F(SplitBitDetectorTest, max_bits_one_below_max) { provider.createBucket(my_bucket, context); - std::vector<spi::DocEntry::UP> entries; + DocEntryList entries; for (uint32_t seed = 0; seed < 10; ++seed) { int location = 1 | (seed % 2 == 0 ? 0x8000 : 0); document::Document::SP doc(testDocMan.createRandomDocumentAtLocation( @@ -114,7 +116,7 @@ TEST_F(SplitBitDetectorTest, max_bits_one_below_max) { } TEST_F(SplitBitDetectorTest, unsplittable) { - std::vector<spi::DocEntry::UP> entries; + DocEntryList entries; for (uint32_t i = 0; i < 10; ++i) { document::Document::SP doc( @@ -130,7 +132,7 @@ TEST_F(SplitBitDetectorTest, unsplittable) { } TEST_F(SplitBitDetectorTest, unsplittable_min_count) { - std::vector<spi::DocEntry::UP> entries; + DocEntryList entries; for (uint32_t i = 0; i < 10; ++i) { document::Document::SP doc( diff --git a/storage/src/tests/persistence/testandsettest.cpp b/storage/src/tests/persistence/testandsettest.cpp index 146dcab2ba7..8cf89b55ad0 100644 --- a/storage/src/tests/persistence/testandsettest.cpp +++ b/storage/src/tests/persistence/testandsettest.cpp @@ -10,6 +10,7 @@ #include <vespa/document/fieldset/fieldsets.h> #include <vespa/persistence/spi/test.h> #include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/persistence/spi/docentry.h> #include <functional> using std::unique_ptr; @@ -73,7 +74,7 @@ struct TestAndSetTest : PersistenceTestUtils { static std::string expectedDocEntryString( api::Timestamp timestamp, const document::DocumentId & testDocId, - spi::DocumentMetaFlags removeFlag = spi::NONE); + spi::DocumentMetaEnum removeFlag = spi::DocumentMetaEnum::NONE); }; TEST_F(TestAndSetTest, conditional_put_not_executed_on_condition_mismatch) { @@ -150,7 +151,7 @@ TEST_F(TestAndSetTest, conditional_remove_executed_on_condition_match) { ASSERT_EQ(fetchResult(asyncHandler->handleRemove(*remove, createTracker(remove, BUCKET))).getResult(), api::ReturnCode::Result::OK); EXPECT_EQ(expectedDocEntryString(timestampOne, testDocId) + - expectedDocEntryString(timestampTwo, testDocId, spi::REMOVE_ENTRY), + expectedDocEntryString(timestampTwo, testDocId, spi::DocumentMetaEnum::REMOVE_ENTRY), dumpBucket(BUCKET_ID)); } @@ -291,12 +292,12 @@ void TestAndSetTest::assertTestDocumentFoundAndMatchesContent(const document::Fi std::string TestAndSetTest::expectedDocEntryString( api::Timestamp timestamp, const document::DocumentId & docId, - spi::DocumentMetaFlags removeFlag) + spi::DocumentMetaEnum removeFlag) { std::stringstream ss; - ss << "DocEntry(" << timestamp << ", " << removeFlag << ", "; - if (removeFlag == spi::REMOVE_ENTRY) { + ss << "DocEntry(" << timestamp << ", " << int(removeFlag) << ", "; + if (removeFlag == spi::DocumentMetaEnum::REMOVE_ENTRY) { ss << docId << ")\n"; } else { ss << "Doc(" << docId << "))\n"; diff --git a/storage/src/tests/visiting/visitortest.cpp b/storage/src/tests/visiting/visitortest.cpp index 494af8a0eff..945a08d910e 100644 --- a/storage/src/tests/visiting/visitortest.cpp +++ b/storage/src/tests/visiting/visitortest.cpp @@ -16,6 +16,7 @@ #include <tests/common/teststorageapp.h> #include <tests/common/dummystoragelink.h> #include <tests/storageserver/testvisitormessagesession.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/gtest/gtest.h> #include <thread> @@ -23,6 +24,8 @@ using namespace std::chrono_literals; using document::test::makeBucketSpace; +using document::Document; +using document::DocumentId; using namespace ::testing; namespace storage { @@ -54,7 +57,7 @@ struct TestParams { struct VisitorTest : Test { static uint32_t docCount; - std::vector<document::Document::SP > _documents; + std::vector<Document::SP> _documents; std::unique_ptr<TestVisitorMessageSessionFactory> _messageSessionFactory; std::unique_ptr<TestServiceLayerApp> _node; std::unique_ptr<DummyStorageLink> _top; @@ -92,11 +95,11 @@ struct VisitorTest : Test { void getMessagesAndReply( int expectedCount, TestVisitorMessageSession& session, - std::vector<document::Document::SP >& docs, - std::vector<document::DocumentId>& docIds, + std::vector<Document::SP> & docs, + std::vector<DocumentId>& docIds, std::vector<std::string>& infoMessages, api::ReturnCode::Result returnCode = api::ReturnCode::OK); - uint32_t getMatchingDocuments(std::vector<document::Document::SP >& docs); + uint32_t getMatchingDocuments(std::vector<Document::SP>& docs); protected: void doTestVisitorInstanceHasConsistencyLevel( @@ -212,7 +215,7 @@ VisitorTest::initializeTest(const TestParams& params) uri << "id:test:testdoctype1:n=" << i % 10 << ":http://www.ntnu.no/" << i << ".html"; - _documents.push_back(document::Document::SP( + _documents.push_back(Document::SP( _node->getTestDocMan().createDocument(content, uri.str()))); const document::DocumentType& type(_documents.back()->getType()); _documents.back()->setValue(type.getField("headerval"), @@ -275,8 +278,8 @@ void VisitorTest::getMessagesAndReply( int expectedCount, TestVisitorMessageSession& session, - std::vector<document::Document::SP >& docs, - std::vector<document::DocumentId>& docIds, + std::vector<Document::SP >& docs, + std::vector<DocumentId>& docIds, std::vector<std::string>& infoMessages, api::ReturnCode::Result result) { @@ -351,7 +354,7 @@ VisitorTest::verifyCreateVisitorReply( } uint32_t -VisitorTest::getMatchingDocuments(std::vector<document::Document::SP >& docs) { +VisitorTest::getMatchingDocuments(std::vector<Document::SP >& docs) { uint32_t equalCount = 0; for (uint32_t i=0; i<docs.size(); ++i) { for (uint32_t j=0; j<_documents.size(); ++j) { @@ -381,11 +384,7 @@ VisitorTest::sendGetIterReply(GetIterCommand& cmd, assert(maxDocuments < _documents.size()); size_t documentCount = maxDocuments != 0 ? maxDocuments : _documents.size(); for (size_t i = 0; i < documentCount; ++i) { - reply->getEntries().emplace_back( - std::make_unique<spi::DocEntry>( - spi::Timestamp(1000 + i), - spi::NONE, - document::Document::UP(_documents[i]->clone()))); + reply->getEntries().push_back(spi::DocEntry::create(spi::Timestamp(1000 + i), Document::UP(_documents[i]->clone()))); } if (documentCount == _documents.size() || overrideCompleted) { reply->setCompleted(); @@ -481,8 +480,8 @@ TEST_F(VisitorTest, normal_usage) { sendGetIterReply(*getIterCmd); - std::vector<document::Document::SP> docs; - std::vector<document::DocumentId> docIds; + std::vector<Document::SP> docs; + std::vector<DocumentId> docIds; std::vector<std::string> infoMessages; getMessagesAndReply(_documents.size(), getSession(0), docs, docIds, infoMessages); ASSERT_EQ(0, infoMessages.size()); @@ -547,8 +546,8 @@ TEST_F(VisitorTest, document_api_client_error) { sendGetIterReply(*getIterCmd, api::ReturnCode(api::ReturnCode::OK), 1); } - std::vector<document::Document::SP> docs; - std::vector<document::DocumentId> docIds; + std::vector<Document::SP> docs; + std::vector<DocumentId> docIds; std::vector<std::string> infoMessages; getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages, api::ReturnCode::INTERNAL_FAILURE); @@ -587,8 +586,8 @@ TEST_F(VisitorTest, no_document_api_resending_for_failed_visitor) { sendGetIterReply(*getIterCmd, api::ReturnCode(api::ReturnCode::OK), 2, true); } - std::vector<document::Document::SP> docs; - std::vector<document::DocumentId> docIds; + std::vector<Document::SP> docs; + std::vector<DocumentId> docIds; std::vector<std::string> infoMessages; // Use non-critical result. Visitor info message should be received // after we send a NOT_CONNECTED reply. Failing this message as well @@ -690,8 +689,8 @@ TEST_F(VisitorTest, no_visitor_notification_for_transient_failures) { ASSERT_NO_FATAL_FAILURE(initializeTest()); ASSERT_NO_FATAL_FAILURE(sendInitialCreateVisitorAndGetIterRound()); - std::vector<document::Document::SP> docs; - std::vector<document::DocumentId> docIds; + std::vector<Document::SP> docs; + std::vector<DocumentId> docIds; std::vector<std::string> infoMessages; // Have to make sure time increases in visitor thread so that resend // times are reached. @@ -734,8 +733,8 @@ TEST_F(VisitorTest, notification_sent_if_transient_error_retried_many_times) { ASSERT_NO_FATAL_FAILURE(initializeTest()); sendInitialCreateVisitorAndGetIterRound(); - std::vector<document::Document::SP> docs; - std::vector<document::DocumentId> docIds; + std::vector<Document::SP> docs; + std::vector<DocumentId> docIds; std::vector<std::string> infoMessages; // Have to make sure time increases in visitor thread so that resend // times are reached. @@ -774,8 +773,8 @@ VisitorTest::doCompleteVisitingSession( 1, true); - std::vector<document::Document::SP> docs; - std::vector<document::DocumentId> docIds; + std::vector<Document::SP> docs; + std::vector<DocumentId> docIds; std::vector<std::string> infoMessages; getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages); @@ -835,8 +834,8 @@ TEST_F(VisitorTest, no_more_iterators_sent_while_memory_used_above_limit) { std::this_thread::sleep_for(100ms); ASSERT_EQ(0, _bottom->getNumCommands()); - std::vector<document::Document::SP> docs; - std::vector<document::DocumentId> docIds; + std::vector<Document::SP> docs; + std::vector<DocumentId> docIds; std::vector<std::string> infoMessages; getMessagesAndReply(1, getSession(0), docs, docIds, infoMessages); @@ -898,8 +897,8 @@ struct ReindexingVisitorTest : VisitorTest { void respond_to_client_put(api::ReturnCode::Result result) { // Reply to the Put from "client" back to the visitor - std::vector<document::Document::SP> docs; - std::vector<document::DocumentId> doc_ids; + std::vector<Document::SP> docs; + std::vector<DocumentId> doc_ids; std::vector<std::string> info_messages; getMessagesAndReply(1, getSession(0), docs, doc_ids, info_messages, result); } diff --git a/storage/src/vespa/storage/common/dummy_mbus_messages.h b/storage/src/vespa/storage/common/dummy_mbus_messages.h new file mode 100644 index 00000000000..10ecf7b6ed7 --- /dev/null +++ b/storage/src/vespa/storage/common/dummy_mbus_messages.h @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/messagebus/message.h> + +/** + * Dummy-implementation of mbus::Message and mbus::Reply to be used when interacting with + * MessageBus IThrottlePolicy subclasses, as these expect message instances as parameters. + */ + +namespace storage { + +template <typename Base> +class DummyMbusMessage : public Base { + static const mbus::string NAME; +public: + const mbus::string& getProtocol() const override { return NAME; } + uint32_t getType() const override { return 0x1badb007; } + uint8_t priority() const override { return 255; } +}; + +template <typename Base> +const mbus::string DummyMbusMessage<Base>::NAME = "FooBar"; + +class DummyMbusRequest final : public DummyMbusMessage<mbus::Message> { +public: + // getApproxSize() returns 1 by default. + // Approximate size of messages allowed by throttle policy is implicitly added to + // internal StaticThrottlePolicy pending size tracking and associated with the + // internal mbus context of the message. + // Since we have no connection between the request and reply instances used when + // interacting with the policy, we have to make sure they cancel each other out + // (i.e. += 0, -= 0). + // Not doing this would cause the StaticThrottlePolicy to keep adding a single byte + // of pending size for each message allowed by the policy. + uint32_t getApproxSize() const override { return 0; } +}; + +class DummyMbusReply final : public DummyMbusMessage<mbus::Reply> {}; + +} diff --git a/storage/src/vespa/storage/config/stor-distributormanager.def b/storage/src/vespa/storage/config/stor-distributormanager.def index 5162e337f24..dba36d1b73d 100644 --- a/storage/src/vespa/storage/config/stor-distributormanager.def +++ b/storage/src/vespa/storage/config/stor-distributormanager.def @@ -189,7 +189,7 @@ sequence_mutating_operations bool default=true ## Number of seconds that scheduling of new merge operations should be inhibited ## towards a node if it has indicated that its merge queues are full or it is ## suffering from resource exhaustion. -inhibit_merge_sending_on_busy_node_duration_sec int default=10 +inhibit_merge_sending_on_busy_node_duration_sec int default=1 ## If set, enables potentially stale reads during cluster state transitions where ## buckets change ownership. This also implicitly enables support for two-phase @@ -285,11 +285,11 @@ num_distributor_stripes int default=0 restart ## bucket maintenance priority database even when no operation can be started for the ## bucket due to being blocked by concurrent operations. This avoids potential head-of-line ## blocking of later buckets in the priority database. -implicitly_clear_bucket_priority_on_schedule bool default=false +implicitly_clear_bucket_priority_on_schedule bool default=true ## Enables sending merges that are forwarded between content nodes in ideal state node key ## order, instead of strictly increasing node key order (which is the default). ## Even if this config is set to true, unordered merges will only be sent if _all_ nodes ## involved in a given merge have previously reported (as part of bucket info fetching) ## that they support the unordered merge feature. -use_unordered_merge_chaining bool default=false +use_unordered_merge_chaining bool default=true diff --git a/storage/src/vespa/storage/config/stor-server.def b/storage/src/vespa/storage/config/stor-server.def index 6611c3cba91..a368c2e5b6f 100644 --- a/storage/src/vespa/storage/config/stor-server.def +++ b/storage/src/vespa/storage/config/stor-server.def @@ -34,7 +34,7 @@ node_reliability int default=1 restart ## merge, only actually starting the operation when every node has ## allowed it to pass through. max_merges_per_node int default=16 -max_merge_queue_size int default=1024 +max_merge_queue_size int default=100 ## If the persistence provider indicates that it has exhausted one or more ## of its internal resources during a mutating operation, new merges will @@ -51,7 +51,7 @@ resource_exhaustion_merge_back_pressure_duration_secs double default=30.0 ## the current queue size. This avoids wasting the time spent being accepted ## into merge windows, which would happen if the merge were to be bounced with ## a busy-reply that would subsequently be unwound through the entire merge chain. -disable_queue_limits_for_chained_merges bool default=false +disable_queue_limits_for_chained_merges bool default=true ## Whether the deadlock detector should be enabled or not. If disabled, it will ## still run, but it will never actually abort the process it is running in. diff --git a/storage/src/vespa/storage/persistence/CMakeLists.txt b/storage/src/vespa/storage/persistence/CMakeLists.txt index c737d2bed28..5e068236026 100644 --- a/storage/src/vespa/storage/persistence/CMakeLists.txt +++ b/storage/src/vespa/storage/persistence/CMakeLists.txt @@ -14,6 +14,7 @@ vespa_add_library(storage_spersistence OBJECT persistenceutil.cpp processallhandler.cpp provider_error_wrapper.cpp + shared_operation_throttler.cpp simplemessagehandler.cpp splitbitdetector.cpp splitjoinhandler.cpp diff --git a/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.cpp b/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.cpp index 2dc5989e857..73ccc7f6085 100644 --- a/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.cpp +++ b/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.cpp @@ -7,10 +7,16 @@ namespace storage { -ApplyBucketDiffEntryComplete::ApplyBucketDiffEntryComplete(std::shared_ptr<ApplyBucketDiffState> state, document::DocumentId doc_id, const char *op, const framework::Clock& clock, metrics::DoubleAverageMetric& latency_metric) +ApplyBucketDiffEntryComplete::ApplyBucketDiffEntryComplete(std::shared_ptr<ApplyBucketDiffState> state, + document::DocumentId doc_id, + SharedOperationThrottler::Token throttle_token, + const char *op, + const framework::Clock& clock, + metrics::DoubleAverageMetric& latency_metric) : _result_handler(nullptr), _state(std::move(state)), _doc_id(std::move(doc_id)), + _throttle_token(std::move(throttle_token)), _op(op), _start_time(clock), _latency_metric(latency_metric) @@ -27,6 +33,7 @@ ApplyBucketDiffEntryComplete::onComplete(std::unique_ptr<spi::Result> result) no } double elapsed = _start_time.getElapsedTimeAsDouble(); _latency_metric.addValue(elapsed); + _throttle_token.reset(); _state->on_entry_complete(std::move(result), _doc_id, _op); } diff --git a/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.h b/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.h index 1037318aec6..8478cab4c17 100644 --- a/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.h +++ b/storage/src/vespa/storage/persistence/apply_bucket_diff_entry_complete.h @@ -2,6 +2,7 @@ #pragma once +#include "shared_operation_throttler.h" #include <vespa/document/base/documentid.h> #include <vespa/metrics/valuemetric.h> #include <vespa/persistence/spi/operationcomplete.h> @@ -21,12 +22,17 @@ class ApplyBucketDiffEntryComplete : public spi::OperationComplete const spi::ResultHandler* _result_handler; std::shared_ptr<ApplyBucketDiffState> _state; document::DocumentId _doc_id; + SharedOperationThrottler::Token _throttle_token; const char* _op; framework::MilliSecTimer _start_time; metrics::DoubleAverageMetric& _latency_metric; public: - ApplyBucketDiffEntryComplete(std::shared_ptr<ApplyBucketDiffState> state, document::DocumentId doc_id, const char *op, const framework::Clock& clock, metrics::DoubleAverageMetric& latency_metric); - ~ApplyBucketDiffEntryComplete(); + ApplyBucketDiffEntryComplete(std::shared_ptr<ApplyBucketDiffState> state, + document::DocumentId doc_id, + SharedOperationThrottler::Token throttle_token, + const char *op, const framework::Clock& clock, + metrics::DoubleAverageMetric& latency_metric); + ~ApplyBucketDiffEntryComplete() override; void onComplete(std::unique_ptr<spi::Result> result) noexcept override; void addResultHandler(const spi::ResultHandler* resultHandler) override; }; diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index b5161673af3..3d24ee87879 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -6,6 +6,7 @@ #include "bucketownershipnotifier.h" #include "bucketprocessor.h" #include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/persistence/spi/catchresult.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/document/update/documentupdate.h> diff --git a/storage/src/vespa/storage/persistence/bucketprocessor.h b/storage/src/vespa/storage/persistence/bucketprocessor.h index c605a9338e3..002cba6c15c 100644 --- a/storage/src/vespa/storage/persistence/bucketprocessor.h +++ b/storage/src/vespa/storage/persistence/bucketprocessor.h @@ -7,11 +7,14 @@ #pragma once #include <vespa/persistence/spi/bucket.h> -#include <vespa/persistence/spi/docentry.h> #include <vespa/persistence/spi/context.h> +#include <persistence/spi/types.h> namespace document { class FieldSet; } -namespace storage::spi { struct PersistenceProvider; } +namespace storage::spi { + struct PersistenceProvider; + class DocEntry; +} namespace storage { diff --git a/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt b/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt index 62d1a80501a..2d137f87118 100644 --- a/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt +++ b/storage/src/vespa/storage/persistence/filestorage/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(storage_filestorpersistence OBJECT SOURCES active_operations_metrics.cpp active_operations_stats.cpp + filestorhandler.cpp filestorhandlerimpl.cpp filestormanager.cpp filestormetrics.cpp diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp new file mode 100644 index 00000000000..c066277ec71 --- /dev/null +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp @@ -0,0 +1,8 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "filestorhandler.h" + +namespace storage { + +FileStorHandler::LockedMessage::~LockedMessage() = default; + +} diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index a980b5aa2e1..6f740ce2c28 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -16,6 +16,7 @@ #include <vespa/document/bucket/bucket.h> #include <vespa/storage/storageutil/resumeguard.h> #include <vespa/storage/common/messagesender.h> +#include <vespa/storage/persistence/shared_operation_throttler.h> #include <vespa/storageapi/messageapi/storagemessage.h> namespace storage { @@ -74,7 +75,29 @@ public: [[nodiscard]] virtual api::LockingRequirements lockingRequirements() const noexcept = 0; }; - using LockedMessage = std::pair<BucketLockInterface::SP, api::StorageMessage::SP>; + struct LockedMessage { + std::shared_ptr<BucketLockInterface> lock; + std::shared_ptr<api::StorageMessage> msg; + SharedOperationThrottler::Token throttle_token; + + LockedMessage() noexcept = default; + LockedMessage(std::shared_ptr<BucketLockInterface> lock_, + std::shared_ptr<api::StorageMessage> msg_) noexcept + : lock(std::move(lock_)), + msg(std::move(msg_)), + throttle_token() + {} + LockedMessage(std::shared_ptr<BucketLockInterface> lock_, + std::shared_ptr<api::StorageMessage> msg_, + SharedOperationThrottler::Token token) noexcept + : lock(std::move(lock_)), + msg(std::move(msg_)), + throttle_token(std::move(token)) + {} + LockedMessage(LockedMessage&&) noexcept = default; + ~LockedMessage(); + }; + class ScheduleAsyncResult { private: bool _was_scheduled; @@ -90,7 +113,7 @@ public: return _was_scheduled; } bool has_async_message() const { - return _async_message.first.get() != nullptr; + return _async_message.lock.get() != nullptr; } const LockedMessage& async_message() const { return _async_message; @@ -250,6 +273,8 @@ public: virtual std::string dumpQueue() const = 0; virtual ActiveOperationsStats get_active_operations_stats(bool reset_min_max) const = 0; + + virtual SharedOperationThrottler& operation_throttler() const noexcept = 0; }; } // storage diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index c6991803b4d..2ccbc7a85ef 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -40,16 +40,18 @@ uint32_t per_stripe_merge_limit(uint32_t num_threads, uint32_t num_stripes) noex FileStorHandlerImpl::FileStorHandlerImpl(MessageSender& sender, FileStorMetrics& metrics, ServiceLayerComponentRegister& compReg) - : FileStorHandlerImpl(1, 1, sender, metrics, compReg) + : FileStorHandlerImpl(1, 1, sender, metrics, compReg, SharedOperationThrottler::make_unlimited_throttler()) { } FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripes, MessageSender& sender, FileStorMetrics& metrics, - ServiceLayerComponentRegister& compReg) + ServiceLayerComponentRegister& compReg, + std::unique_ptr<SharedOperationThrottler> operation_throttler) : _component(compReg, "filestorhandlerimpl"), _state(FileStorHandler::AVAILABLE), _metrics(nullptr), + _operation_throttler(std::move(operation_throttler)), _stripes(), _messageSender(sender), _bucketIdFactory(_component.getBucketIdFactory()), @@ -330,6 +332,7 @@ FileStorHandlerImpl::updateMetrics(const MetricLockGuard &) std::lock_guard lockGuard(_mergeStatesLock); _metrics->pendingMerges.addValue(_mergeStates.size()); _metrics->queueSize.addValue(getQueueSize()); + _metrics->throttle_window_size.addValue(_operation_throttler->current_window_size()); for (const auto & stripe : _metrics->stripes) { const auto & m = stripe->averageQueueWaitingTime; @@ -885,10 +888,39 @@ FileStorHandlerImpl::Stripe::Stripe(const FileStorHandlerImpl & owner, MessageSe _active_operations_stats() {} +namespace { + +bool +operation_type_should_be_throttled(api::MessageType::Id type_id) noexcept +{ + // Note: SetBucketState is intentionally _not_ included in this set, even though it's + // dispatched async. The rationale behind this is that SetBucketState is very cheap + // to execute, usually comes in large waves (up to #buckets count) and processing all + // requests should complete as quickly as possible. We also don't want such waves to + // artificially boost the dynamic throttle window size due to a sudden throughput spike. + // + // Merge-related operations are transitively throttled by using the operation throttler + // directly for all async ops within the MergeHandler. + switch (type_id) { + case api::MessageType::PUT_ID: + case api::MessageType::REMOVE_ID: + case api::MessageType::UPDATE_ID: + case api::MessageType::REMOVELOCATION_ID: + case api::MessageType::CREATEBUCKET_ID: + case api::MessageType::DELETEBUCKET_ID: + return true; + default: + return false; + } +} + +} + FileStorHandler::LockedMessage FileStorHandlerImpl::Stripe::getNextMessage(vespalib::duration timeout) { std::unique_lock guard(*_lock); + SharedOperationThrottler::Token throttle_token; // Try to grab a message+lock, immediately retrying once after a wait // if none can be found and then exiting if the same is the case on the // second attempt. This is key to allowing the run loop to register @@ -896,15 +928,43 @@ FileStorHandlerImpl::Stripe::getNextMessage(vespalib::duration timeout) for (int attempt = 0; (attempt < 2) && !_owner.isPaused(); ++attempt) { PriorityIdx& idx(bmi::get<1>(*_queue)); PriorityIdx::iterator iter(idx.begin()), end(idx.end()); + bool was_throttled = false; - while (iter != end && operationIsInhibited(guard, iter->_bucket, *iter->_command)) { + while ((iter != end) && operationIsInhibited(guard, iter->_bucket, *iter->_command)) { iter++; } if (iter != end) { - return getMessage(guard, idx, iter); + const bool should_throttle_op = operation_type_should_be_throttled(iter->_command->getType().getId()); + if (!should_throttle_op && throttle_token.valid()) { + throttle_token.reset(); // Let someone else play with it. + } else if (should_throttle_op && !throttle_token.valid()) { + // Important: _non-blocking_ attempt at getting a throttle token. + throttle_token = _owner.operation_throttler().try_acquire_one(); + if (!throttle_token.valid()) { + was_throttled = true; + _metrics->throttled_persistence_thread_polls.inc(); + } + } + if (!should_throttle_op || throttle_token.valid()) { + return getMessage(guard, idx, iter, std::move(throttle_token)); + } } if (attempt == 0) { - _cond->wait_for(guard, timeout); + // Depending on whether we were blocked due to no usable ops in queue or throttling, + // wait for either the queue or throttler to (hopefully) have some fresh stuff for us. + if (!was_throttled) { + _cond->wait_for(guard, timeout); + } else { + // Have to release lock before doing a blocking throttle token fetch, since it + // prevents RPC threads from pushing onto the queue. + guard.unlock(); + throttle_token = _owner.operation_throttler().blocking_acquire_one(timeout); + guard.lock(); + if (!throttle_token.valid()) { + _metrics->timeouts_waiting_for_throttle_token.inc(); + return {}; // Already exhausted our timeout window. + } + } } } return {}; // No message fetched. @@ -923,14 +983,22 @@ FileStorHandlerImpl::Stripe::get_next_async_message(monitor_guard& guard) ++iter; } if ((iter != end) && AsyncHandler::is_async_message(iter->_command->getType().getId())) { - return getMessage(guard, idx, iter); + // This is executed in the context of an RPC thread, so only do a _non-blocking_ + // poll of the throttle policy. + auto throttle_token = _owner.operation_throttler().try_acquire_one(); + if (throttle_token.valid()) { + return getMessage(guard, idx, iter, std::move(throttle_token)); + } else { + _metrics->throttled_rpc_direct_dispatches.inc(); + } } return {}; } FileStorHandler::LockedMessage -FileStorHandlerImpl::Stripe::getMessage(monitor_guard & guard, PriorityIdx & idx, PriorityIdx::iterator iter) { - +FileStorHandlerImpl::Stripe::getMessage(monitor_guard & guard, PriorityIdx & idx, PriorityIdx::iterator iter, + SharedOperationThrottler::Token throttle_token) +{ std::chrono::milliseconds waitTime(uint64_t(iter->_timer.stop(_metrics->averageQueueWaitingTime))); std::shared_ptr<api::StorageMessage> msg = std::move(iter->_command); @@ -942,7 +1010,7 @@ FileStorHandlerImpl::Stripe::getMessage(monitor_guard & guard, PriorityIdx & idx msg->getType().getId(), msg->getMsgId(), msg->lockingRequirements()); guard.unlock(); - return FileStorHandler::LockedMessage(std::move(locker), std::move(msg)); + return {std::move(locker), std::move(msg), std::move(throttle_token)}; } else { std::shared_ptr<api::StorageReply> msgReply(makeQueueTimeoutReply(*msg)); guard.unlock(); @@ -1014,7 +1082,7 @@ FileStorHandlerImpl::Stripe::schedule_and_get_next_async_message(MessageEntry en std::unique_lock guard(*_lock); _queue->emplace_back(std::move(entry)); auto lockedMessage = get_next_async_message(guard); - if ( ! lockedMessage.second) { + if ( ! lockedMessage.msg) { if (guard.owns_lock()) { guard.unlock(); } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index 5d68be8a800..c4b85ac596c 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -42,11 +42,12 @@ class AbortBucketOperationsCommand; namespace bmi = boost::multi_index; -class FileStorHandlerImpl : private framework::MetricUpdateHook, - private ResumeGuard::Callback, - public FileStorHandler { +class FileStorHandlerImpl final + : private framework::MetricUpdateHook, + private ResumeGuard::Callback, + public FileStorHandler +{ public: - struct MessageEntry { std::shared_ptr<api::StorageMessage> _command; metrics::MetricTimer _timer; @@ -147,7 +148,8 @@ public: // Precondition: the bucket used by `iter`s operation is not locked in a way that conflicts // with its locking requirements. FileStorHandler::LockedMessage getMessage(monitor_guard & guard, PriorityIdx & idx, - PriorityIdx::iterator iter); + PriorityIdx::iterator iter, + SharedOperationThrottler::Token throttle_token); using LockedBuckets = vespalib::hash_map<document::Bucket, MultiLockEntry, document::Bucket::hash>; const FileStorHandlerImpl &_owner; MessageSender &_messageSender; @@ -163,7 +165,9 @@ public: class BucketLock : public FileStorHandler::BucketLockInterface { public: // TODO refactor, too many params - BucketLock(const monitor_guard & guard, Stripe& disk, const document::Bucket &bucket, + BucketLock(const monitor_guard & guard, + Stripe& disk, + const document::Bucket &bucket, uint8_t priority, api::MessageType::Id msgType, api::StorageMessage::Id, api::LockingRequirements lockReq); ~BucketLock() override; @@ -187,9 +191,9 @@ public: FileStorHandlerImpl(MessageSender& sender, FileStorMetrics& metrics, ServiceLayerComponentRegister& compReg); FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripes, MessageSender&, FileStorMetrics&, - ServiceLayerComponentRegister&); + ServiceLayerComponentRegister&, std::unique_ptr<SharedOperationThrottler>); - ~FileStorHandlerImpl(); + ~FileStorHandlerImpl() override; void setGetNextMessageTimeout(vespalib::duration timeout) override { _getNextMessageTimeout = timeout; } void flush(bool killPendingMerges) override; @@ -239,6 +243,10 @@ public: ResumeGuard pause() override; void abortQueuedOperations(const AbortBucketOperationsCommand& cmd) override; + SharedOperationThrottler& operation_throttler() const noexcept override { + return *_operation_throttler; + } + // Implements ResumeGuard::Callback void resume() override; @@ -249,6 +257,7 @@ private: ServiceLayerComponent _component; std::atomic<DiskState> _state; FileStorDiskMetrics * _metrics; + std::unique_ptr<SharedOperationThrottler> _operation_throttler; std::vector<Stripe> _stripes; MessageSender& _messageSender; const document::BucketIdFactory& _bucketIdFactory; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index 2cfb3a2cffe..f5b9da0e1f5 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -32,6 +32,7 @@ LOG_SETUP(".persistence.filestor.manager"); using std::shared_ptr; using document::BucketSpace; using vespalib::make_string_short::fmt; +using vespa::config::content::StorFilestorConfig; namespace { @@ -130,18 +131,31 @@ uint32_t computeNumResponseThreads(int configured) { } vespalib::Executor::OptimizeFor -selectSequencer(vespa::config::content::StorFilestorConfig::ResponseSequencerType sequencerType) { +selectSequencer(StorFilestorConfig::ResponseSequencerType sequencerType) { switch (sequencerType) { - case vespa::config::content::StorFilestorConfig::ResponseSequencerType::THROUGHPUT: + case StorFilestorConfig::ResponseSequencerType::THROUGHPUT: return vespalib::Executor::OptimizeFor::THROUGHPUT; - case vespa::config::content::StorFilestorConfig::ResponseSequencerType::LATENCY: + case StorFilestorConfig::ResponseSequencerType::LATENCY: return vespalib::Executor::OptimizeFor::LATENCY; - case vespa::config::content::StorFilestorConfig::ResponseSequencerType::ADAPTIVE: + case StorFilestorConfig::ResponseSequencerType::ADAPTIVE: default: return vespalib::Executor::OptimizeFor::ADAPTIVE; } } +std::unique_ptr<SharedOperationThrottler> +make_operation_throttler_from_config(const StorFilestorConfig& config, size_t num_threads) +{ + const bool use_dynamic_throttling = (config.asyncOperationThrottlerType == StorFilestorConfig::AsyncOperationThrottlerType::DYNAMIC); + if (use_dynamic_throttling) { + auto config_win_size_incr = std::max(config.asyncOperationDynamicThrottlingWindowIncrement, 1); + auto win_size_increment = std::max(static_cast<size_t>(config_win_size_incr), num_threads); + return SharedOperationThrottler::make_dynamic_throttler(win_size_increment); + } else { + return SharedOperationThrottler::make_unlimited_throttler(); + } +} + #ifdef __PIC__ #define TLS_LINKAGE __attribute__((visibility("hidden"), tls_model("initial-exec"))) #else @@ -185,7 +199,7 @@ FileStorManager::getThreadLocalHandler() { * incoming during reconfiguration */ void -FileStorManager::configure(std::unique_ptr<vespa::config::content::StorFilestorConfig> config) +FileStorManager::configure(std::unique_ptr<StorFilestorConfig> config) { // If true, this is not the first configure. bool liveUpdate = ! _threads.empty(); @@ -198,8 +212,10 @@ FileStorManager::configure(std::unique_ptr<vespa::config::content::StorFilestorC size_t numThreads = _config->numThreads; size_t numStripes = std::max(size_t(1u), numThreads / 2); _metrics->initDiskMetrics(numStripes, computeAllPossibleHandlerThreads(*_config)); + auto operation_throttler = make_operation_throttler_from_config(*_config, numThreads); - _filestorHandler = std::make_unique<FileStorHandlerImpl>(numThreads, numStripes, *this, *_metrics, _compReg); + _filestorHandler = std::make_unique<FileStorHandlerImpl>(numThreads, numStripes, *this, *_metrics, + _compReg, std::move(operation_throttler)); uint32_t numResponseThreads = computeNumResponseThreads(_config->numResponseThreads); _sequencedExecutor = vespalib::SequencedTaskExecutor::create(response_executor, numResponseThreads, 10000, selectSequencer(_config->responseSequencerType)); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp index c119fdc4f69..6cb76c32997 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp @@ -190,7 +190,16 @@ FileStorThreadMetrics::~FileStorThreadMetrics() = default; FileStorStripeMetrics::FileStorStripeMetrics(const std::string& name, const std::string& description) : MetricSet(name, {{"partofsum"}}, description), - averageQueueWaitingTime("averagequeuewait", {}, "Average time an operation spends in input queue.", this) + averageQueueWaitingTime("averagequeuewait", {}, "Average time an operation spends in input queue.", this), + throttled_rpc_direct_dispatches("throttled_rpc_direct_dispatches", {}, + "Number of times an RPC thread could not directly dispatch an async operation " + "directly to Proton because it was disallowed by the throttle policy", this), + throttled_persistence_thread_polls("throttled_persistence_thread_polls", {}, + "Number of times a persistence thread could not immediately dispatch a " + "queued async operation because it was disallowed by the throttle policy", this), + timeouts_waiting_for_throttle_token("timeouts_waiting_for_throttle_token", {}, + "Number of times a persistence thread timed out waiting for an available " + "throttle policy token", this) { } @@ -203,6 +212,7 @@ FileStorDiskMetrics::FileStorDiskMetrics(const std::string& name, const std::str averageQueueWaitingTime("averagequeuewait.sum", {}, "Average time an operation spends in input queue.", this), queueSize("queuesize", {}, "Size of input message queue.", this), pendingMerges("pendingmerge", {}, "Number of buckets currently being merged.", this), + throttle_window_size("throttlewindowsize", {}, "Current size of async operation throttler window size", this), waitingForLockHitRate("waitingforlockrate", {}, "Amount of times a filestor thread has needed to wait for " "lock to take next message in queue.", this), diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h index 7543e6e0771..1bcbacd2ca6 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h @@ -131,6 +131,9 @@ class FileStorStripeMetrics : public metrics::MetricSet public: using SP = std::shared_ptr<FileStorStripeMetrics>; metrics::DoubleAverageMetric averageQueueWaitingTime; + metrics::LongCountMetric throttled_rpc_direct_dispatches; + metrics::LongCountMetric throttled_persistence_thread_polls; + metrics::LongCountMetric timeouts_waiting_for_throttle_token; FileStorStripeMetrics(const std::string& name, const std::string& description); ~FileStorStripeMetrics() override; }; @@ -147,6 +150,7 @@ public: metrics::DoubleAverageMetric averageQueueWaitingTime; metrics::LongAverageMetric queueSize; metrics::LongAverageMetric pendingMerges; + metrics::LongAverageMetric throttle_window_size; metrics::DoubleAverageMetric waitingForLockHitRate; metrics::DoubleAverageMetric lockWaitTime; // unused ActiveOperationsMetrics active_operations; diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 0c9cecdb6a1..7dcf4bcbee2 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -2,11 +2,12 @@ #include "mergehandler.h" #include "persistenceutil.h" +#include "shared_operation_throttler.h" #include "apply_bucket_diff_entry_complete.h" #include "apply_bucket_diff_state.h" #include <vespa/storage/persistence/filestorage/mergestatus.h> #include <vespa/persistence/spi/persistenceprovider.h> -#include <vespa/persistence/spi/catchresult.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/document/fieldset/fieldsets.h> #include <vespa/vespalib/objects/nbostream.h> @@ -32,6 +33,7 @@ MergeHandler::MergeHandler(PersistenceUtil& env, spi::PersistenceProvider& spi, _cluster_context(cluster_context), _env(env), _spi(spi), + _operation_throttler(_env._fileStorHandler.operation_throttler()), _monitored_ref_count(std::make_unique<MonitoredRefCount>()), _maxChunkSize(maxChunkSize), _commonMergeChainOptimalizationMinimumSize(commonMergeChainOptimalizationMinimumSize), @@ -130,11 +132,8 @@ void update_op_metrics(FileStorThreadMetrics& metrics, const api::StorageReply & } // anonymous namespace void -MergeHandler::populateMetaData( - const spi::Bucket& bucket, - Timestamp maxTimestamp, - std::vector<spi::DocEntry::UP>& entries, - spi::Context& context) const +MergeHandler::populateMetaData(const spi::Bucket& bucket, Timestamp maxTimestamp, + DocEntryList& entries, spi::Context& context) const { spi::DocumentSelection docSel(""); @@ -150,9 +149,7 @@ MergeHandler::populateMetaData( if (createIterResult.getErrorCode() != spi::Result::ErrorType::NONE) { std::ostringstream ss; ss << "Failed to create iterator for " - << bucket - << ": " - << createIterResult.getErrorMessage(); + << bucket << ": " << createIterResult.getErrorMessage(); throw std::runtime_error(ss.str()); } spi::IteratorId iteratorId(createIterResult.getIteratorId()); @@ -163,9 +160,7 @@ MergeHandler::populateMetaData( if (result.getErrorCode() != spi::Result::ErrorType::NONE) { std::ostringstream ss; ss << "Failed to iterate for " - << bucket - << ": " - << result.getErrorMessage(); + << bucket << ": " << result.getErrorMessage(); throw std::runtime_error(ss.str()); } auto list = result.steal_entries(); @@ -241,7 +236,7 @@ MergeHandler::buildBucketInfoList( } } - std::vector<spi::DocEntry::UP> entries; + DocEntryList entries; populateMetaData(bucket, maxTimestamp, entries, context); for (const auto& entry : entries) { @@ -402,7 +397,7 @@ MergeHandler::fetchLocalData( IteratorGuard iteratorGuard(_spi, iteratorId, context); // Fetch all entries - std::vector<spi::DocEntry::UP> entries; + DocEntryList entries; entries.reserve(slots.size()); bool fetchedAllLocalData = false; bool chunkLimitReached = false; @@ -521,17 +516,22 @@ MergeHandler::applyDiffEntry(std::shared_ptr<ApplyBucketDiffState> async_results spi::Context& context, const document::DocumentTypeRepo& repo) const { + auto throttle_token = _operation_throttler.blocking_acquire_one(); spi::Timestamp timestamp(e._entry._timestamp); if (!(e._entry._flags & (DELETED | DELETED_IN_PLACE))) { // Regular put entry Document::SP doc(deserializeDiffDocument(e, repo)); DocumentId docId = doc->getId(); - auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), std::move(docId), "put", _clock, _env._metrics.merge_handler_metrics.put_latency); + auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), std::move(docId), + std::move(throttle_token), "put", + _clock, _env._metrics.merge_handler_metrics.put_latency); _spi.putAsync(bucket, timestamp, std::move(doc), context, std::move(complete)); } else { std::vector<spi::PersistenceProvider::TimeStampAndDocumentId> ids; ids.emplace_back(timestamp, e._docName); - auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), ids[0].second, "remove", _clock, _env._metrics.merge_handler_metrics.remove_latency); + auto complete = std::make_unique<ApplyBucketDiffEntryComplete>(std::move(async_results), ids[0].second, + std::move(throttle_token), "remove", + _clock, _env._metrics.merge_handler_metrics.remove_latency); _spi.removeAsync(bucket, std::move(ids), context, std::move(complete)); } } @@ -557,7 +557,7 @@ MergeHandler::applyDiffLocally( uint32_t notNeededByteCount = 0; async_results->mark_stale_bucket_info(); - std::vector<spi::DocEntry::UP> entries; + DocEntryList entries; populateMetaData(bucket, MAX_TIMESTAMP, entries, context); const document::DocumentTypeRepo & repo = _env.getDocumentTypeRepo(); diff --git a/storage/src/vespa/storage/persistence/mergehandler.h b/storage/src/vespa/storage/persistence/mergehandler.h index 5e66b364242..1007f35c241 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.h +++ b/storage/src/vespa/storage/persistence/mergehandler.h @@ -16,7 +16,6 @@ #include "types.h" #include "merge_bucket_info_syncer.h" #include <vespa/persistence/spi/bucket.h> -#include <vespa/persistence/spi/docentry.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storage/common/cluster_context.h> #include <vespa/storage/common/messagesender.h> @@ -30,11 +29,12 @@ namespace storage { namespace spi { struct PersistenceProvider; class Context; + class DocEntry; } class PersistenceUtil; -class ApplyBucketDiffEntryResult; class ApplyBucketDiffState; class MergeStatus; +class SharedOperationThrottler; class MergeHandler : public Types, public MergeBucketInfoSyncer { @@ -53,7 +53,7 @@ public: uint32_t commonMergeChainOptimalizationMinimumSize = 64, bool async_apply_bucket_diff = false); - ~MergeHandler(); + ~MergeHandler() override; bool buildBucketInfoList( const spi::Bucket& bucket, @@ -82,10 +82,12 @@ public: void configure(bool async_apply_bucket_diff) noexcept; private: + using DocEntryList = std::vector<std::unique_ptr<spi::DocEntry>>; const framework::Clock &_clock; const ClusterContext &_cluster_context; PersistenceUtil &_env; spi::PersistenceProvider &_spi; + SharedOperationThrottler& _operation_throttler; std::unique_ptr<vespalib::MonitoredRefCount> _monitored_ref_count; const uint32_t _maxChunkSize; const uint32_t _commonMergeChainOptimalizationMinimumSize; @@ -117,7 +119,7 @@ private: */ void populateMetaData(const spi::Bucket&, Timestamp maxTimestamp, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList & entries, spi::Context& context) const; Document::UP deserializeDiffDocument( diff --git a/storage/src/vespa/storage/persistence/messages.cpp b/storage/src/vespa/storage/persistence/messages.cpp index c6f7442cfdc..cf05ccd65b8 100644 --- a/storage/src/vespa/storage/persistence/messages.cpp +++ b/storage/src/vespa/storage/persistence/messages.cpp @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "messages.h" +#include <vespa/persistence/spi/docentry.h> #include <ostream> -#include <cassert> using document::BucketSpace; diff --git a/storage/src/vespa/storage/persistence/messages.h b/storage/src/vespa/storage/persistence/messages.h index 594506f53ba..86cf1ac3b06 100644 --- a/storage/src/vespa/storage/persistence/messages.h +++ b/storage/src/vespa/storage/persistence/messages.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/storageapi/message/internal.h> -#include <vespa/persistence/spi/docentry.h> #include <vespa/persistence/spi/bucket.h> #include <vespa/persistence/spi/selection.h> #include <vespa/persistence/spi/read_consistency.h> @@ -10,6 +9,8 @@ namespace storage { +namespace spi { class DocEntry; } + class GetIterCommand : public api::InternalCommand { private: document::Bucket _bucket; @@ -44,9 +45,10 @@ private: class GetIterReply : public api::InternalReply { private: + using List = std::vector<std::unique_ptr<spi::DocEntry>>; document::Bucket _bucket; - std::vector<spi::DocEntry::UP> _entries; - bool _completed; + List _entries; + bool _completed; public: typedef std::unique_ptr<GetIterReply> UP; @@ -58,13 +60,9 @@ public: document::Bucket getBucket() const override { return _bucket; } - const std::vector<spi::DocEntry::UP>& getEntries() const { - return _entries; - } + const List & getEntries() const { return _entries; } - std::vector<spi::DocEntry::UP>& getEntries() { - return _entries; - } + List & getEntries() { return _entries; } void setCompleted(bool completed = true) { _completed = completed; } bool isCompleted() const { return _completed; } diff --git a/storage/src/vespa/storage/persistence/persistencehandler.cpp b/storage/src/vespa/storage/persistence/persistencehandler.cpp index 8b546771b71..c0c95ffd7af 100644 --- a/storage/src/vespa/storage/persistence/persistencehandler.cpp +++ b/storage/src/vespa/storage/persistence/persistencehandler.cpp @@ -158,12 +158,13 @@ PersistenceHandler::processMessage(api::StorageMessage& msg, MessageTracker::UP void PersistenceHandler::processLockedMessage(FileStorHandler::LockedMessage lock) const { - LOG(debug, "NodeIndex %d, ptr=%p", _env._nodeIndex, lock.second.get()); - api::StorageMessage & msg(*lock.second); + LOG(debug, "NodeIndex %d, ptr=%p", _env._nodeIndex, lock.msg.get()); + api::StorageMessage & msg(*lock.msg); // Important: we _copy_ the message shared_ptr instead of moving to ensure that `msg` remains // valid even if the tracker is destroyed by an exception in processMessage(). - auto tracker = std::make_unique<MessageTracker>(framework::MilliSecTimer(_clock), _env, _env._fileStorHandler, std::move(lock.first), lock.second); + auto tracker = std::make_unique<MessageTracker>(framework::MilliSecTimer(_clock), _env, _env._fileStorHandler, + std::move(lock.lock), lock.msg, std::move(lock.throttle_token)); tracker = processMessage(msg, std::move(tracker)); if (tracker) { tracker->sendReply(); diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index f9da4d63d7f..499e9807cbf 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -38,7 +38,7 @@ PersistenceThread::run(framework::ThreadHandle& thread) FileStorHandler::LockedMessage lock(_fileStorHandler.getNextMessage(_stripeId)); - if (lock.first) { + if (lock.lock) { _persistenceHandler.processLockedMessage(std::move(lock)); } } diff --git a/storage/src/vespa/storage/persistence/persistenceutil.cpp b/storage/src/vespa/storage/persistence/persistenceutil.cpp index cbfc9463a8c..65eab99b8fb 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.cpp +++ b/storage/src/vespa/storage/persistence/persistenceutil.cpp @@ -31,19 +31,22 @@ MessageTracker::MessageTracker(const framework::MilliSecTimer & timer, const PersistenceUtil & env, MessageSender & replySender, FileStorHandler::BucketLockInterface::SP bucketLock, - api::StorageMessage::SP msg) - : MessageTracker(timer, env, replySender, true, std::move(bucketLock), std::move(msg)) + api::StorageMessage::SP msg, + SharedOperationThrottler::Token throttle_token) + : MessageTracker(timer, env, replySender, true, std::move(bucketLock), std::move(msg), std::move(throttle_token)) {} MessageTracker::MessageTracker(const framework::MilliSecTimer & timer, const PersistenceUtil & env, MessageSender & replySender, bool updateBucketInfo, FileStorHandler::BucketLockInterface::SP bucketLock, - api::StorageMessage::SP msg) + api::StorageMessage::SP msg, + SharedOperationThrottler::Token throttle_token) : _sendReply(true), _updateBucketInfo(updateBucketInfo && hasBucketInfo(msg->getType().getId())), _bucketLock(std::move(bucketLock)), _msg(std::move(msg)), + _throttle_token(std::move(throttle_token)), _context(_msg->getPriority(), _msg->getTrace().getLevel()), _env(env), _replySender(replySender), @@ -56,7 +59,8 @@ MessageTracker::UP MessageTracker::createForTesting(const framework::MilliSecTimer & timer, PersistenceUtil &env, MessageSender &replySender, FileStorHandler::BucketLockInterface::SP bucketLock, api::StorageMessage::SP msg) { - return MessageTracker::UP(new MessageTracker(timer, env, replySender, false, std::move(bucketLock), std::move(msg))); + return MessageTracker::UP(new MessageTracker(timer, env, replySender, false, std::move(bucketLock), + std::move(msg), SharedOperationThrottler::Token())); } void diff --git a/storage/src/vespa/storage/persistence/persistenceutil.h b/storage/src/vespa/storage/persistence/persistenceutil.h index 4fd0e60c730..588cbef2170 100644 --- a/storage/src/vespa/storage/persistence/persistenceutil.h +++ b/storage/src/vespa/storage/persistence/persistenceutil.h @@ -30,7 +30,8 @@ public: using UP = std::unique_ptr<MessageTracker>; MessageTracker(const framework::MilliSecTimer & timer, const PersistenceUtil & env, MessageSender & replySender, - FileStorHandler::BucketLockInterface::SP bucketLock, std::shared_ptr<api::StorageMessage> msg); + FileStorHandler::BucketLockInterface::SP bucketLock, std::shared_ptr<api::StorageMessage> msg, + SharedOperationThrottler::Token throttle_token); ~MessageTracker(); @@ -91,7 +92,8 @@ public: private: MessageTracker(const framework::MilliSecTimer & timer, const PersistenceUtil & env, MessageSender & replySender, bool updateBucketInfo, - FileStorHandler::BucketLockInterface::SP bucketLock, std::shared_ptr<api::StorageMessage> msg); + FileStorHandler::BucketLockInterface::SP bucketLock, std::shared_ptr<api::StorageMessage> msg, + SharedOperationThrottler::Token throttle_token); [[nodiscard]] bool count_result_as_failure() const noexcept; @@ -99,6 +101,7 @@ private: bool _updateBucketInfo; FileStorHandler::BucketLockInterface::SP _bucketLock; std::shared_ptr<api::StorageMessage> _msg; + SharedOperationThrottler::Token _throttle_token; spi::Context _context; const PersistenceUtil &_env; MessageSender &_replySender; diff --git a/storage/src/vespa/storage/persistence/processallhandler.cpp b/storage/src/vespa/storage/persistence/processallhandler.cpp index 6a2b5e79450..6d6723a0185 100644 --- a/storage/src/vespa/storage/persistence/processallhandler.cpp +++ b/storage/src/vespa/storage/persistence/processallhandler.cpp @@ -5,6 +5,7 @@ #include "persistenceutil.h" #include <vespa/document/fieldset/fieldsets.h> #include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> @@ -31,7 +32,7 @@ public: if (e.getDocument() != nullptr) { ost << "Doc(" << e.getDocument()->getId() << ")" << ", " << e.getDocument()->getId().getGlobalId().toString() - << ", size: " << e.getPersistedDocumentSize(); + << ", size: " << e.getSize(); } else if (e.getDocumentId() != nullptr) { ost << *e.getDocumentId() << ", " << e.getDocumentId()->getGlobalId().toString(); diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp index 752c1175f22..b7344098698 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp @@ -2,6 +2,7 @@ #include "provider_error_wrapper.h" #include "persistenceutil.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/vespalib/util/idestructorcallback.h> namespace storage { diff --git a/storage/src/vespa/storage/persistence/shared_operation_throttler.cpp b/storage/src/vespa/storage/persistence/shared_operation_throttler.cpp new file mode 100644 index 00000000000..7b05decb851 --- /dev/null +++ b/storage/src/vespa/storage/persistence/shared_operation_throttler.cpp @@ -0,0 +1,199 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "shared_operation_throttler.h" +#include <vespa/messagebus/dynamicthrottlepolicy.h> +#include <vespa/storage/common/dummy_mbus_messages.h> +#include <condition_variable> +#include <cassert> +#include <mutex> + +namespace storage { + +namespace { + +class NoLimitsOperationThrottler final : public SharedOperationThrottler { +public: + ~NoLimitsOperationThrottler() override = default; + Token blocking_acquire_one() noexcept override { + return Token(this, TokenCtorTag{}); + } + Token blocking_acquire_one(vespalib::duration) noexcept override { + return Token(this, TokenCtorTag{}); + } + Token try_acquire_one() noexcept override { + return Token(this, TokenCtorTag{}); + } + uint32_t current_window_size() const noexcept override { return 0; } + uint32_t waiting_threads() const noexcept override { return 0; } +private: + void release_one() noexcept override { /* no-op */ } +}; + +class DynamicOperationThrottler final : public SharedOperationThrottler { + mutable std::mutex _mutex; + std::condition_variable _cond; + mbus::DynamicThrottlePolicy _throttle_policy; + uint32_t _pending_ops; + uint32_t _waiting_threads; +public: + explicit DynamicOperationThrottler(uint32_t min_size_and_window_increment); + ~DynamicOperationThrottler() override; + + Token blocking_acquire_one() noexcept override; + Token blocking_acquire_one(vespalib::duration timeout) noexcept override; + Token try_acquire_one() noexcept override; + uint32_t current_window_size() const noexcept override; + uint32_t waiting_threads() const noexcept override; +private: + void release_one() noexcept override; + // Non-const since actually checking the send window of a dynamic throttler might change + // it if enough time has passed. + [[nodiscard]] bool has_spare_capacity_in_active_window() noexcept; + void add_one_to_active_window_size(); + void subtract_one_from_active_window_size(); +}; + +DynamicOperationThrottler::DynamicOperationThrottler(uint32_t min_size_and_window_increment) + : _mutex(), + _cond(), + _throttle_policy(static_cast<double>(min_size_and_window_increment)), + _pending_ops(0), + _waiting_threads(0) +{ +} + +DynamicOperationThrottler::~DynamicOperationThrottler() = default; + +bool +DynamicOperationThrottler::has_spare_capacity_in_active_window() noexcept +{ + DummyMbusRequest dummy_request; + return _throttle_policy.canSend(dummy_request, _pending_ops); +} + +void +DynamicOperationThrottler::add_one_to_active_window_size() +{ + DummyMbusRequest dummy_request; + _throttle_policy.processMessage(dummy_request); + ++_pending_ops; +} + +void +DynamicOperationThrottler::subtract_one_from_active_window_size() +{ + DummyMbusReply dummy_reply; + _throttle_policy.processReply(dummy_reply); + assert(_pending_ops > 0); + --_pending_ops; +} + +DynamicOperationThrottler::Token +DynamicOperationThrottler::blocking_acquire_one() noexcept +{ + std::unique_lock lock(_mutex); + if (!has_spare_capacity_in_active_window()) { + ++_waiting_threads; + _cond.wait(lock, [&] { + return has_spare_capacity_in_active_window(); + }); + --_waiting_threads; + } + add_one_to_active_window_size(); + return Token(this, TokenCtorTag{}); +} + +DynamicOperationThrottler::Token +DynamicOperationThrottler::blocking_acquire_one(vespalib::duration timeout) noexcept +{ + std::unique_lock lock(_mutex); + if (!has_spare_capacity_in_active_window()) { + ++_waiting_threads; + const bool accepted = _cond.wait_for(lock, timeout, [&] { + return has_spare_capacity_in_active_window(); + }); + --_waiting_threads; + if (!accepted) { + return Token(); + } + } + add_one_to_active_window_size(); + return Token(this, TokenCtorTag{}); +} + +DynamicOperationThrottler::Token +DynamicOperationThrottler::try_acquire_one() noexcept +{ + std::unique_lock lock(_mutex); + if (!has_spare_capacity_in_active_window()) { + return Token(); + } + add_one_to_active_window_size(); + return Token(this, TokenCtorTag{}); +} + +void +DynamicOperationThrottler::release_one() noexcept +{ + std::unique_lock lock(_mutex); + subtract_one_from_active_window_size(); + // Only wake up a waiting thread if doing so would possibly result in success. + if ((_waiting_threads > 0) && has_spare_capacity_in_active_window()) { + lock.unlock(); + _cond.notify_one(); + } +} + +uint32_t +DynamicOperationThrottler::current_window_size() const noexcept +{ + std::unique_lock lock(_mutex); + return _throttle_policy.getMaxPendingCount(); // Actually returns current window size +} + +uint32_t +DynamicOperationThrottler::waiting_threads() const noexcept +{ + std::unique_lock lock(_mutex); + return _waiting_threads; +} + +} + +std::unique_ptr<SharedOperationThrottler> +SharedOperationThrottler::make_unlimited_throttler() +{ + return std::make_unique<NoLimitsOperationThrottler>(); +} + +std::unique_ptr<SharedOperationThrottler> +SharedOperationThrottler::make_dynamic_throttler(uint32_t min_size_and_window_increment) +{ + return std::make_unique<DynamicOperationThrottler>(min_size_and_window_increment); +} + +DynamicOperationThrottler::Token::~Token() +{ + if (_throttler) { + _throttler->release_one(); + } +} + +void +DynamicOperationThrottler::Token::reset() noexcept +{ + if (_throttler) { + _throttler->release_one(); + _throttler = nullptr; + } +} + +DynamicOperationThrottler::Token& +DynamicOperationThrottler::Token::operator=(Token&& rhs) noexcept +{ + reset(); + _throttler = rhs._throttler; + rhs._throttler = nullptr; + return *this; +} + +} diff --git a/storage/src/vespa/storage/persistence/shared_operation_throttler.h b/storage/src/vespa/storage/persistence/shared_operation_throttler.h new file mode 100644 index 00000000000..4ee8d017c05 --- /dev/null +++ b/storage/src/vespa/storage/persistence/shared_operation_throttler.h @@ -0,0 +1,72 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/vespalib/util/time.h> +#include <memory> +#include <optional> + +namespace storage { + +/** + * Operation throttler that is intended to provide global throttling of + * async operations across all persistence stripe threads. A throttler + * wraps a logical max pending window size of in-flight operations. Depending + * on the throttler implementation, the window size may expand and shrink + * dynamically. Exactly how and when this happens is unspecified. + * + * Offers both polling and (timed, non-timed) blocking calls for acquiring + * a throttle token. If the returned token is valid, the caller may proceed + * to invoke the asynchronous operation. + * + * The window slot taken up by a valid throttle token is implicitly freed up + * when the token is destroyed. + * + * All operations on the throttler are thread safe. + */ +class SharedOperationThrottler { +protected: + struct TokenCtorTag {}; // Make available to subclasses for token construction. +public: + class Token { + SharedOperationThrottler* _throttler; + public: + constexpr Token(SharedOperationThrottler* throttler, TokenCtorTag) noexcept : _throttler(throttler) {} + constexpr Token() noexcept : _throttler(nullptr) {} + constexpr Token(Token&& rhs) noexcept + : _throttler(rhs._throttler) + { + rhs._throttler = nullptr; + } + Token& operator=(Token&& rhs) noexcept; + ~Token(); + + Token(const Token&) = delete; + Token& operator=(const Token&) = delete; + + [[nodiscard]] constexpr bool valid() const noexcept { return (_throttler != nullptr); } + void reset() noexcept; + }; + + virtual ~SharedOperationThrottler() = default; + + // All methods are thread safe + [[nodiscard]] virtual Token blocking_acquire_one() noexcept = 0; + [[nodiscard]] virtual Token blocking_acquire_one(vespalib::duration timeout) noexcept = 0; + [[nodiscard]] virtual Token try_acquire_one() noexcept = 0; + + // May return 0, in which case the window size is unlimited. + [[nodiscard]] virtual uint32_t current_window_size() const noexcept = 0; + + // Exposed for unit testing only. + [[nodiscard]] virtual uint32_t waiting_threads() const noexcept = 0; + + // Creates a throttler that does exactly zero throttling (but also has zero overhead and locking) + static std::unique_ptr<SharedOperationThrottler> make_unlimited_throttler(); + // Creates a throttler that uses a MessageBus DynamicThrottlePolicy under the hood + static std::unique_ptr<SharedOperationThrottler> make_dynamic_throttler(uint32_t min_size_and_window_increment); +private: + // Exclusively called from a valid Token. Thread safe. + virtual void release_one() noexcept = 0; +}; + +} diff --git a/storage/src/vespa/storage/persistence/simplemessagehandler.cpp b/storage/src/vespa/storage/persistence/simplemessagehandler.cpp index 9a7a451b906..74813e2e891 100644 --- a/storage/src/vespa/storage/persistence/simplemessagehandler.cpp +++ b/storage/src/vespa/storage/persistence/simplemessagehandler.cpp @@ -3,6 +3,7 @@ #include "simplemessagehandler.h" #include "persistenceutil.h" #include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/document/base/exceptions.h> #include <vespa/document/fieldset/fieldsetrepo.h> diff --git a/storage/src/vespa/storage/persistence/splitbitdetector.cpp b/storage/src/vespa/storage/persistence/splitbitdetector.cpp index bd472868e06..2a9ac635cff 100644 --- a/storage/src/vespa/storage/persistence/splitbitdetector.cpp +++ b/storage/src/vespa/storage/persistence/splitbitdetector.cpp @@ -3,6 +3,7 @@ #include "splitbitdetector.h" #include "bucketprocessor.h" #include <vespa/persistence/spi/persistenceprovider.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/document/base/documentid.h> #include <vespa/document/fieldset/fieldsets.h> diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index bc2f54e5a50..2a30acb1a74 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -2,6 +2,7 @@ #include "mergethrottler.h" #include <vespa/storage/common/nodestateupdater.h> +#include <vespa/storage/common/dummy_mbus_messages.h> #include <vespa/storage/persistence/messages.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/messagebus/message.h> @@ -27,22 +28,6 @@ struct NodeComparator { } }; -// Class used to sneakily get around IThrottlePolicy only accepting -// messagebus objects -template <typename Base> -class DummyMbusMessage : public Base { -private: - static const mbus::string NAME; -public: - const mbus::string& getProtocol() const override { return NAME; } - uint32_t getType() const override { return 0x1badb007; } - - uint8_t priority() const override { return 255; } -}; - -template <typename Base> -const mbus::string DummyMbusMessage<Base>::NAME = "SkyNet"; - } MergeThrottler::ChainedMergeState::ChainedMergeState() @@ -310,7 +295,7 @@ MergeThrottler::onFlush(bool /*downwards*/) "own the command", merge.first.toString().c_str()); } - DummyMbusMessage<mbus::Reply> dummyReply; + DummyMbusReply dummyReply; _throttlePolicy->processReply(dummyReply); } for (auto& entry : _queue) { @@ -419,7 +404,7 @@ MergeThrottler::enqueue_merge_for_later_processing( bool MergeThrottler::canProcessNewMerge() const { - DummyMbusMessage<mbus::Message> dummyMsg; + DummyMbusRequest dummyMsg; return _throttlePolicy->canSend(dummyMsg, _merges.size()); } @@ -858,7 +843,7 @@ MergeThrottler::processNewMergeCommand( LOG(debug, "Added merge %s to internal state", mergeCmd.toString().c_str()); - DummyMbusMessage<mbus::Message> dummyMsg; + DummyMbusRequest dummyMsg; _throttlePolicy->processMessage(dummyMsg); bool execute = false; @@ -1058,7 +1043,7 @@ MergeThrottler::processMergeReply( updateOperationMetrics(mergeReply.getResult(), _metrics->local); } - DummyMbusMessage<mbus::Reply> dummyReply; + DummyMbusReply dummyReply; if (mergeReply.getResult().failed()) { // Must be sure to add an error if reply contained a failure, since // DynamicThrottlePolicy penalizes on failed transmissions diff --git a/storage/src/vespa/storage/visiting/countvisitor.cpp b/storage/src/vespa/storage/visiting/countvisitor.cpp index e20699aa799..3971544a9a0 100644 --- a/storage/src/vespa/storage/visiting/countvisitor.cpp +++ b/storage/src/vespa/storage/visiting/countvisitor.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "countvisitor.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/vespalib/util/stringfmt.h> @@ -22,7 +23,7 @@ CountVisitor::CountVisitor(StorageComponent& component, void CountVisitor::handleDocuments(const document::BucketId& /*bucketId*/, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList& entries, HitCounter& hitCounter) { for (size_t i = 0; i < entries.size(); ++i) { diff --git a/storage/src/vespa/storage/visiting/countvisitor.h b/storage/src/vespa/storage/visiting/countvisitor.h index 4c436b3d07c..4120a96225f 100644 --- a/storage/src/vespa/storage/visiting/countvisitor.h +++ b/storage/src/vespa/storage/visiting/countvisitor.h @@ -18,11 +18,11 @@ public: CountVisitor(StorageComponent&, const vdslib::Parameters& params); - virtual void completedVisiting(HitCounter&) override; + void completedVisiting(HitCounter&) override; private: void handleDocuments(const document::BucketId& bucketId, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& hitCounter) override; bool _doScheme; diff --git a/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp b/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp index 704aaa219cb..3419d329a06 100644 --- a/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp +++ b/storage/src/vespa/storage/visiting/dumpvisitorsingle.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "dumpvisitorsingle.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/document/update/documentupdate.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> @@ -16,8 +17,8 @@ DumpVisitorSingle::DumpVisitorSingle(StorageComponent& component, const vdslib:: { } -void DumpVisitorSingle::handleDocuments(const document::BucketId& /*bucketId*/, - std::vector<spi::DocEntry::UP>& entries, +void DumpVisitorSingle::handleDocuments(const document::BucketId&, + DocEntryList& entries, HitCounter& hitCounter) { LOG(debug, "Visitor %s handling block of %zu documents.", @@ -25,7 +26,7 @@ void DumpVisitorSingle::handleDocuments(const document::BucketId& /*bucketId*/, for (size_t i = 0; i < entries.size(); ++i) { spi::DocEntry& entry(*entries[i]); - const uint32_t docSize = entry.getDocumentSize(); + const uint32_t docSize = entry.getSize(); if (entry.isRemove()) { hitCounter.addHit(*entry.getDocumentId(), docSize); sendMessage(std::make_unique<documentapi::RemoveDocumentMessage>(*entry.getDocumentId())); diff --git a/storage/src/vespa/storage/visiting/dumpvisitorsingle.h b/storage/src/vespa/storage/visiting/dumpvisitorsingle.h index 81f4ab7e989..c98bad17e84 100644 --- a/storage/src/vespa/storage/visiting/dumpvisitorsingle.h +++ b/storage/src/vespa/storage/visiting/dumpvisitorsingle.h @@ -19,7 +19,7 @@ public: const vdslib::Parameters& params); private: - void handleDocuments(const document::BucketId&, std::vector<spi::DocEntry::UP>&, HitCounter&) override; + void handleDocuments(const document::BucketId&, DocEntryList&, HitCounter&) override; }; struct DumpVisitorSingleFactory : public VisitorFactory { diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp index df965f93ae6..1c3c97d7dcb 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.cpp +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.cpp @@ -2,6 +2,7 @@ #include "recoveryvisitor.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/documentapi/messagebus/messages/visitor.h> #include <vespa/vespalib/text/stringtokenizer.h> @@ -31,7 +32,7 @@ RecoveryVisitor::RecoveryVisitor(StorageComponent& component, void RecoveryVisitor::handleDocuments(const document::BucketId& bid, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& hitCounter) { std::lock_guard guard(_mutex); diff --git a/storage/src/vespa/storage/visiting/recoveryvisitor.h b/storage/src/vespa/storage/visiting/recoveryvisitor.h index 045c8aeb484..e850eca3f37 100644 --- a/storage/src/vespa/storage/visiting/recoveryvisitor.h +++ b/storage/src/vespa/storage/visiting/recoveryvisitor.h @@ -22,7 +22,7 @@ public: private: void handleDocuments(const document::BucketId& bucketId, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& hitCounter) override; void completedBucket(const document::BucketId&, HitCounter&) override; diff --git a/storage/src/vespa/storage/visiting/reindexing_visitor.cpp b/storage/src/vespa/storage/visiting/reindexing_visitor.cpp index 528f83e29cc..0b08c52bdc4 100644 --- a/storage/src/vespa/storage/visiting/reindexing_visitor.cpp +++ b/storage/src/vespa/storage/visiting/reindexing_visitor.cpp @@ -3,6 +3,7 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/documentapi/messagebus/messages/putdocumentmessage.h> #include <vespa/storage/common/reindexing_constants.h> +#include <vespa/persistence/spi/docentry.h> #include <vespa/log/log.h> LOG_SETUP(".visitor.instance.reindexing_visitor"); @@ -14,8 +15,8 @@ ReindexingVisitor::ReindexingVisitor(StorageComponent& component) { } -void ReindexingVisitor::handleDocuments(const document::BucketId& /*bucketId*/, - std::vector<spi::DocEntry::UP>& entries, +void ReindexingVisitor::handleDocuments(const document::BucketId& , + DocEntryList & entries, HitCounter& hitCounter) { auto lock_token = make_lock_access_token(); @@ -26,7 +27,7 @@ void ReindexingVisitor::handleDocuments(const document::BucketId& /*bucketId*/, // We don't reindex removed documents, as that would be very silly. continue; } - const uint32_t doc_size = entry->getDocumentSize(); + const uint32_t doc_size = entry->getSize(); hitCounter.addHit(*entry->getDocumentId(), doc_size); auto msg = std::make_unique<documentapi::PutDocumentMessage>(entry->releaseDocument()); msg->setApproxSize(doc_size); diff --git a/storage/src/vespa/storage/visiting/reindexing_visitor.h b/storage/src/vespa/storage/visiting/reindexing_visitor.h index 9bde9903617..d9e18542818 100644 --- a/storage/src/vespa/storage/visiting/reindexing_visitor.h +++ b/storage/src/vespa/storage/visiting/reindexing_visitor.h @@ -20,7 +20,7 @@ public: ~ReindexingVisitor() override = default; private: - void handleDocuments(const document::BucketId&, std::vector<spi::DocEntry::UP>&, HitCounter&) override; + void handleDocuments(const document::BucketId&, DocEntryList&, HitCounter&) override; bool remap_docapi_message_error_code(api::ReturnCode& in_out_code) override; vespalib::string make_lock_access_token() const; }; diff --git a/storage/src/vespa/storage/visiting/testvisitor.cpp b/storage/src/vespa/storage/visiting/testvisitor.cpp index b5ace673e48..10c79ea6f18 100644 --- a/storage/src/vespa/storage/visiting/testvisitor.cpp +++ b/storage/src/vespa/storage/visiting/testvisitor.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "testvisitor.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/documentapi/messagebus/messages/visitor.h> #include <sstream> @@ -39,7 +40,7 @@ TestVisitor::startingVisitor(const std::vector<document::BucketId>& buckets) void TestVisitor::handleDocuments(const document::BucketId& /*bucketId*/, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& /*hitCounter*/) { std::ostringstream ost; diff --git a/storage/src/vespa/storage/visiting/testvisitor.h b/storage/src/vespa/storage/visiting/testvisitor.h index 482b1347faa..989581ac121 100644 --- a/storage/src/vespa/storage/visiting/testvisitor.h +++ b/storage/src/vespa/storage/visiting/testvisitor.h @@ -20,7 +20,7 @@ private: void startingVisitor(const std::vector<document::BucketId>& buckets) override; void handleDocuments(const document::BucketId& bucketId, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& hitCounter) override; void completedBucket(const document::BucketId& bucket, HitCounter& hitCounter) override; diff --git a/storage/src/vespa/storage/visiting/visitor.cpp b/storage/src/vespa/storage/visiting/visitor.cpp index dfb78122e07..b66285f5048 100644 --- a/storage/src/vespa/storage/visiting/visitor.cpp +++ b/storage/src/vespa/storage/visiting/visitor.cpp @@ -2,6 +2,7 @@ #include "visitor.h" #include "visitormetrics.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/storageframework/generic/clock/timer.h> #include <vespa/storageapi/message/datagram.h> #include <vespa/storage/persistence/messages.h> @@ -814,12 +815,11 @@ Visitor::onGetIterReply(const std::shared_ptr<GetIterReply>& reply, uint64_t size = 0; for (const auto& entry : reply->getEntries()) { - size += entry->getPersistedDocumentSize(); + size += entry->getSize(); } _visitorStatistics.setDocumentsVisited( - _visitorStatistics.getDocumentsVisited() - + reply->getEntries().size()); + _visitorStatistics.getDocumentsVisited() + reply->getEntries().size()); _visitorStatistics.setBytesVisited(_visitorStatistics.getBytesVisited() + size); } catch (std::exception& e) { LOG(warning, "handleDocuments threw exception %s", e.what()); diff --git a/storage/src/vespa/storage/visiting/visitor.h b/storage/src/vespa/storage/visiting/visitor.h index 52b2d586c78..8857a54e8df 100644 --- a/storage/src/vespa/storage/visiting/visitor.h +++ b/storage/src/vespa/storage/visiting/visitor.h @@ -18,7 +18,6 @@ #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/common/visitorfactory.h> #include <vespa/documentapi/messagebus/messages/documentmessage.h> -#include <vespa/persistence/spi/docentry.h> #include <vespa/persistence/spi/selection.h> #include <vespa/persistence/spi/read_consistency.h> #include <list> @@ -39,6 +38,10 @@ namespace documentapi { namespace storage { +namespace spi { + class DocEntry; +} + namespace api { class ReturnCode; class StorageCommand; @@ -357,6 +360,7 @@ protected: // error code, false if the DocumentAPI message should be retried later. [[nodiscard]] virtual bool remap_docapi_message_error_code(api::ReturnCode& in_out_code); public: + using DocEntryList = std::vector<std::unique_ptr<spi::DocEntry>>; Visitor(StorageComponent& component); virtual ~Visitor(); @@ -398,7 +402,7 @@ public: * vector of documents arrive from the persistence layer. */ virtual void handleDocuments(const document::BucketId&, - std::vector<spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& hitCounter) = 0; /** diff --git a/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp b/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp index b6244521a46..a936146fd26 100644 --- a/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp +++ b/streamingvisitors/src/tests/searchvisitor/searchvisitor_test.cpp @@ -10,6 +10,8 @@ #include <vespa/searchvisitor/searchvisitor.h> #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> +#include <vespa/persistence/spi/docentry.h> + #include <vespa/log/log.h> LOG_SETUP("searchvisitor_test"); @@ -53,14 +55,13 @@ SearchVisitorTest::SearchVisitorTest() : SearchVisitorTest::~SearchVisitorTest() = default; -std::vector<spi::DocEntry::UP> +Visitor::DocEntryList createDocuments(const vespalib::string & dir) { (void) dir; - std::vector<spi::DocEntry::UP> documents; + Visitor::DocEntryList documents; spi::Timestamp ts; - document::Document::UP doc(new document::Document()); - spi::DocEntry::UP e(new spi::DocEntry(ts, 0, std::move(doc))); + auto e = spi::DocEntry::create(ts, std::make_unique<Document>()); documents.push_back(std::move(e)); return documents; } @@ -72,7 +73,7 @@ SearchVisitorTest::testCreateSearchVisitor(const vespalib::string & dir, const v VisitorFactory & factory(sFactory); std::unique_ptr<Visitor> sv(static_cast<SearchVisitor *>(factory.makeVisitor(*_component, _env, params))); document::BucketId bucketId; - std::vector<spi::DocEntry::UP> documents(createDocuments(dir)); + Visitor::DocEntryList documents(createDocuments(dir)); Visitor::HitCounter hitCounter; sv->handleDocuments(bucketId, documents, hitCounter); } diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index cc6323c3053..460a0886ee2 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -3,6 +3,7 @@ #include "querytermdata.h" #include "searchenvironment.h" #include "searchvisitor.h" +#include <vespa/persistence/spi/docentry.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/datatype/weightedsetdatatype.h> @@ -864,7 +865,7 @@ SearchVisitor::compatibleDocumentTypes(const document::DocumentType& typeA, void SearchVisitor::handleDocuments(const document::BucketId&, - std::vector<storage::spi::DocEntry::UP>& entries, + DocEntryList & entries, HitCounter& hitCounter) { (void) hitCounter; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index dff263b8418..20ab1ccf325 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -298,7 +298,7 @@ private: // Inherit doc from Visitor void handleDocuments(const document::BucketId&, - std::vector<storage::spi::DocEntry::UP>& entries, + DocEntryList& entries, HitCounter& hitCounter) override; bool compatibleDocumentTypes(const document::DocumentType& typeA, diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java index 4b54b392d12..bb62dc51603 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzX509CertificateUtils.java @@ -26,13 +26,29 @@ public class AthenzX509CertificateUtils { private AthenzX509CertificateUtils() {} public static AthenzIdentity getIdentityFromRoleCertificate(X509Certificate certificate) { - List<com.yahoo.security.SubjectAlternativeName> sans = com.yahoo.security.X509CertificateUtils.getSubjectAlternativeNames(certificate); + List<SubjectAlternativeName> sans = X509CertificateUtils.getSubjectAlternativeNames(certificate); + return getRoleIdentityFromEmail(sans) + .or(() -> getRoleIdentityFromUri(sans)) + .orElseThrow(() -> new IllegalArgumentException("Could not find identity in SAN: " + sans)); + } + + private static Optional<AthenzIdentity> getRoleIdentityFromEmail(List<SubjectAlternativeName> sans) { return sans.stream() .filter(san -> san.getType() == RFC822_NAME) .map(com.yahoo.security.SubjectAlternativeName::getValue) .map(AthenzX509CertificateUtils::getIdentityFromSanEmail) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Could not find identity in SAN: " + sans)); + .findFirst(); + } + + private static Optional<AthenzIdentity> getRoleIdentityFromUri(List<SubjectAlternativeName> sans) { + String uriPrefix = "athenz://principal/"; + return sans.stream() + .filter(s -> s.getType() == UNIFORM_RESOURCE_IDENTIFIER && s.getValue().startsWith(uriPrefix)) + .map(san -> { + String uriPath = URI.create(san.getValue()).getPath(); + return AthenzIdentities.from(uriPath.substring(uriPrefix.length())); + }) + .findFirst(); } public static AthenzRole getRolesFromRoleCertificate(X509Certificate certificate) { diff --git a/vespa-hadoop/pom.xml b/vespa-hadoop/pom.xml index d6d9d0e2141..3cf669f924a 100644 --- a/vespa-hadoop/pom.xml +++ b/vespa-hadoop/pom.xml @@ -71,12 +71,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - <version>2.5.0</version> - <scope>provided</scope> - </dependency> - <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> <scope>compile</scope> diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh index f75ffdbd13f..d2677e58a00 100755 --- a/vespabase/src/common-env.sh +++ b/vespabase/src/common-env.sh @@ -123,7 +123,7 @@ export LD_LIBRARY_PATH=$VESPA_HOME/lib64 export MALLOC_ARENA_MAX=1 # Prefer newer gdb and pstack -prepend_path /opt/rh/gcc-toolset-10/root/usr/bin +prepend_path /opt/rh/gcc-toolset-11/root/usr/bin # Maven is needed for tester applications prepend_path "$VESPA_HOME/local/maven/bin" diff --git a/vespaclient-core/pom.xml b/vespaclient-core/pom.xml index 43c8700a06a..8f2865a22cf 100644 --- a/vespaclient-core/pom.xml +++ b/vespaclient-core/pom.xml @@ -29,12 +29,6 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <compilerArgs> - <arg>-Xlint:all</arg> - <arg>-Werror</arg> - </compilerArgs> - </configuration> </plugin> <plugin> <groupId>com.yahoo.vespa</groupId> diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 5eeee267cf6..71e6c0f28f4 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -3026,7 +3026,7 @@ "public void write(char[], int, int)", "public void flush()", "public void close()", - "public final java.io.Writer getWriter()" + "public java.io.Writer getWriter()" ], "fields": [] }, diff --git a/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryPrefix.java b/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryPrefix.java index 64121756e5c..cf5183ec9d3 100644 --- a/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryPrefix.java +++ b/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryPrefix.java @@ -3,9 +3,11 @@ package com.yahoo.binaryprefix; /** * Represents binary prefixes. + * * @author Tony Vaagenes */ public enum BinaryPrefix { + //represents the binary prefix 2^(k*10) unit(0), kilo(1, 'K'), @@ -42,4 +44,5 @@ public enum BinaryPrefix { } throw new RuntimeException("No such binary prefix: " + c); } + } diff --git a/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryScaledAmount.java b/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryScaledAmount.java index ec2886d32bb..fef1f5bac4a 100644 --- a/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryScaledAmount.java +++ b/vespajlib/src/main/java/com/yahoo/binaryprefix/BinaryScaledAmount.java @@ -11,6 +11,7 @@ package com.yahoo.binaryprefix; * @author Tony Vaagenes */ public final class BinaryScaledAmount { + public final double amount; public final BinaryPrefix binaryPrefix; @@ -53,4 +54,5 @@ public final class BinaryScaledAmount { public int hashCode() { return (int)BinaryPrefix.unit.convertFrom(amount, binaryPrefix); } + } diff --git a/vespajlib/src/main/java/com/yahoo/collections/BobHash.java b/vespajlib/src/main/java/com/yahoo/collections/BobHash.java index cb5e88b3131..d133af2ea84 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/BobHash.java +++ b/vespajlib/src/main/java/com/yahoo/collections/BobHash.java @@ -4,22 +4,18 @@ package com.yahoo.collections; import com.yahoo.text.Utf8; /** - * <p>A Java port of Michael Susag's BobHash in FastLib. This version is + * A Java port of Michael Susag's BobHash in FastLib. This version is * specifically done to be bit compatible with the one in FastLib, as it - * is used in decoding packets from FastServer.</p> + * is used in decoding packets from FastServer. * - * <p>Hash function based on - * <a href="http://burtleburtle.net/bob/hash/index.html"> - * http://burtleburtle.net/bob/hash/index.html</a> + * Hash function based on + * <a href="http://burtleburtle.net/bob/hash/index.html">http://burtleburtle.net/bob/hash/index.html</a> * by Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. You may use this - * code any way you wish, private, educational, or commercial. It's free.</p> - * - * @author Michael Susag - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> - * + * code any way you wish, private, educational, or commercial. It's free. * + * @author Michael Susag + * @author Steinar Knutsen */ - public class BobHash { /** diff --git a/vespajlib/src/main/java/com/yahoo/collections/CollectionComparator.java b/vespajlib/src/main/java/com/yahoo/collections/CollectionComparator.java index 5754718b66e..d936bbe41d4 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/CollectionComparator.java +++ b/vespajlib/src/main/java/com/yahoo/collections/CollectionComparator.java @@ -8,7 +8,7 @@ import java.util.Iterator; * Utility class which is useful when implementing <code>Comparable</code> and one needs to * compare Collections of Comparables as instance variables. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class CollectionComparator { /** @@ -51,4 +51,5 @@ public class CollectionComparator { //we haven't returned yet; contents must be equal: return 0; } + } diff --git a/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java b/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java index 1154df1db83..58d39e097d8 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java +++ b/vespajlib/src/main/java/com/yahoo/collections/CollectionUtil.java @@ -15,7 +15,6 @@ import java.util.stream.Collectors; * * @author Tony Vaagenes * @author gjoranv - * @since 5.1.8 */ public class CollectionUtil { @@ -100,4 +99,5 @@ public class CollectionUtil { } return Optional.empty(); } + } diff --git a/vespajlib/src/main/java/com/yahoo/collections/Comparables.java b/vespajlib/src/main/java/com/yahoo/collections/Comparables.java index 5952c5ec012..1f97cdd1ee3 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/Comparables.java +++ b/vespajlib/src/main/java/com/yahoo/collections/Comparables.java @@ -8,6 +8,7 @@ package com.yahoo.collections; * @author hakon */ public class Comparables { + /** * Returns the least element, or {@code first} if they are equal according to * {@link Comparable#compareTo(Object) compareTo}. @@ -23,4 +24,5 @@ public class Comparables { public static <T extends Comparable<? super T>> T max(T first, T second) { return first.compareTo(second) <= 0 ? second : first; } + } diff --git a/vespajlib/src/main/java/com/yahoo/collections/FreezableArrayList.java b/vespajlib/src/main/java/com/yahoo/collections/FreezableArrayList.java index 1a765beaa3b..43ba88cf10c 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/FreezableArrayList.java +++ b/vespajlib/src/main/java/com/yahoo/collections/FreezableArrayList.java @@ -10,7 +10,6 @@ import java.util.List; * Freezable lists may optionally allow new items to be added to the end of the list also after freeze. * * @author bratseth - * @since 5.20 */ public class FreezableArrayList<ITEM> extends ListenableArrayList<ITEM> { diff --git a/vespajlib/src/main/java/com/yahoo/collections/Hashlet.java b/vespajlib/src/main/java/com/yahoo/collections/Hashlet.java index ef31c38cc5a..bfd62a4c9fd 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/Hashlet.java +++ b/vespajlib/src/main/java/com/yahoo/collections/Hashlet.java @@ -20,8 +20,8 @@ package com.yahoo.collections; * maximum load factor is 0.7 and drops slightly with increasing * capacity. * - * @author <a href="mailto:havardpe@yahoo-inc.com">Havard Pettersen</a> - **/ + * @author Havard Pettersen + */ public final class Hashlet<K, V> { private static final int[] emptyHash = new int[1]; @@ -223,4 +223,5 @@ public final class Hashlet<K, V> { } return true; } + } diff --git a/vespajlib/src/main/java/com/yahoo/collections/IntArrayComparator.java b/vespajlib/src/main/java/com/yahoo/collections/IntArrayComparator.java index b4122c3a311..59c0ac5758c 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/IntArrayComparator.java +++ b/vespajlib/src/main/java/com/yahoo/collections/IntArrayComparator.java @@ -5,9 +5,10 @@ package com.yahoo.collections; * Utility class which is useful when implementing <code>Comparable</code> and one needs to * compare int arrays as instance variables. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class IntArrayComparator { + /** * Compare the arguments. Shorter arrays are always considered * smaller than longer arrays. For arrays of equal lengths, the elements @@ -29,7 +30,7 @@ public class IntArrayComparator { return 1; } - //lengths are equal, compare contents + // lengths are equal, compare contents for (int i = 0; i < first.length; i++) { if (first[i] < second[i]) { return -1; @@ -39,7 +40,8 @@ public class IntArrayComparator { //values at index i are equal, continue... } - //we haven't returned yet; contents must be equal: + // we haven't returned yet; contents must be equal: return 0; } + } diff --git a/vespajlib/src/main/java/com/yahoo/collections/ListMap.java b/vespajlib/src/main/java/com/yahoo/collections/ListMap.java index 3f6321cb3d9..809acb69bf5 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/ListMap.java +++ b/vespajlib/src/main/java/com/yahoo/collections/ListMap.java @@ -133,6 +133,7 @@ public class ListMap<K, V> { */ public void freeze() { if (frozen) return; + frozen = true; for (Map.Entry<K,List<V>> entry : map.entrySet()) entry.setValue(ImmutableList.copyOf(entry.getValue())); diff --git a/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java b/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java index 6fbf03925cb..9a807d0fcc7 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java +++ b/vespajlib/src/main/java/com/yahoo/collections/ListenableArrayList.java @@ -11,7 +11,6 @@ import java.util.List; * * @author bratseth */ -@SuppressWarnings("serial") public class ListenableArrayList<ITEM> extends ArrayList<ITEM> { private List<Runnable> listeners = null; diff --git a/vespajlib/src/main/java/com/yahoo/collections/MD5.java b/vespajlib/src/main/java/com/yahoo/collections/MD5.java index de50b1d7410..bc1a8b29c64 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/MD5.java +++ b/vespajlib/src/main/java/com/yahoo/collections/MD5.java @@ -15,9 +15,10 @@ import java.security.NoSuchAlgorithmException; * <p> * This class is not thread safe. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class MD5 { + public static final ThreadLocal<MessageDigest> md5 = new MD5Factory(); private static class MD5Factory extends ThreadLocal<MessageDigest> { @@ -27,6 +28,7 @@ public class MD5 { return createMD5(); } } + private static MessageDigest createMD5() { try { return MessageDigest.getInstance("MD5"); @@ -34,7 +36,9 @@ public class MD5 { throw new RuntimeException(e); } } + final private MessageDigest digester; + public MD5() { try { digester = MessageDigest.getInstance("MD5"); @@ -64,4 +68,5 @@ public class MD5 { public byte[] hashFull(String s) { return digester.digest(Utf8.toBytes(s)); } + } diff --git a/vespajlib/src/main/java/com/yahoo/collections/MethodCache.java b/vespajlib/src/main/java/com/yahoo/collections/MethodCache.java index ef64a2e344a..ea8060e203b 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/MethodCache.java +++ b/vespajlib/src/main/java/com/yahoo/collections/MethodCache.java @@ -10,6 +10,7 @@ import java.lang.reflect.Method; * @author baldersheim */ public final class MethodCache { + private final String methodName; private final CopyOnWriteHashMap<String, Method> cache = new CopyOnWriteHashMap<>(); @@ -34,4 +35,5 @@ public final class MethodCache { return null; } } + } diff --git a/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java b/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java index 831fddd7de8..f9fe365c4c7 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java +++ b/vespajlib/src/main/java/com/yahoo/collections/TinyIdentitySet.java @@ -13,17 +13,12 @@ import java.util.Set; * ArrayList for identity matches. In other words: Performance will only be * acceptable for <i>small</i> sets. * - * <p> * The rationale for this class is the high cost of the object identifier used * in IdentityHashMap, where the key set is often used as an identity set. - * </p> * * @author Steinar Knutsen - * @since 5.1.4 * @see java.util.IdentityHashMap - * - * @param <E> - * the type contained in the Set + * @param <E> the type contained in the Set */ public final class TinyIdentitySet<E> implements Set<E> { private class ArrayIterator<T> implements Iterator<E> { diff --git a/vespajlib/src/main/java/com/yahoo/collections/Tuple2.java b/vespajlib/src/main/java/com/yahoo/collections/Tuple2.java index c95e643a588..4ee02c45efa 100644 --- a/vespajlib/src/main/java/com/yahoo/collections/Tuple2.java +++ b/vespajlib/src/main/java/com/yahoo/collections/Tuple2.java @@ -2,15 +2,13 @@ package com.yahoo.collections; /** - * A representation of a pair of values, typically of different types. + * A pair of values. * - * <p> * This class is to avoid littering a class with thin wrapper objects for * passing around e.g. the state of an operation and the result value. Using * this class may be correct, but it is a symptom that you may want to redesign * your code. (Should you pass mutable objects to the method instead? Create a * new class and do the work inside that class instead? Etc.) - * </p> * * @author Steinar Knutsen */ diff --git a/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java index 50022256478..dc4132a3f3f 100644 --- a/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java +++ b/vespajlib/src/main/java/com/yahoo/compress/CompressionType.java @@ -14,7 +14,7 @@ public enum CompressionType { LZ4((byte) 6), ZSTD((byte) 7); - private byte code; + private final byte code; CompressionType(byte code) { this.code = code; diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java b/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java index 13d2463cf2c..c693d46975f 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/CachedThreadPoolWithFallback.java @@ -13,9 +13,9 @@ import java.util.concurrent.TimeUnit; /** * An executor that will first try a bounded cached thread pool before falling back to an unbounded * single threaded thread pool that will take over dispatching to the primary pool. - * */ public class CachedThreadPoolWithFallback implements AutoCloseable, Executor { + private final ExecutorService primary; private final ExecutorService secondary; diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java b/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java index 3e84970764a..c1a3ee30b9a 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/DaemonThreadFactory.java @@ -11,7 +11,8 @@ import java.util.concurrent.ThreadFactory; * @author Einar M R Rosenvinge */ public class DaemonThreadFactory implements ThreadFactory { - private ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory(); + + private final ThreadFactory defaultThreadFactory = Executors.defaultThreadFactory(); private String prefix = null; /** diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/LocalInstance.java b/vespajlib/src/main/java/com/yahoo/concurrent/LocalInstance.java index e52e80963f3..98a509a89c2 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/LocalInstance.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/LocalInstance.java @@ -10,14 +10,12 @@ import com.yahoo.concurrent.ThreadLocalDirectory.Updater; * {@link ThreadLocal} in ThreadLocalDirectory if possible, but has no user * available methods. * - * @param <AGGREGATOR> - * the structure to insert produced data into - * @param <SAMPLE> - * type of produced data to insert from each participating thread - * + * @param <AGGREGATOR> the structure to insert produced data into + * @param <SAMPLE> type of produced data to insert from each participating thread * @author Steinar Knutsen */ public final class LocalInstance<AGGREGATOR, SAMPLE> { + /** * The current generation of data produced from a single thread, where * generation is the period between two subsequent calls to diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/Receiver.java b/vespajlib/src/main/java/com/yahoo/concurrent/Receiver.java index ace822e071f..7f8aa4b8725 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/Receiver.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/Receiver.java @@ -27,7 +27,7 @@ import com.yahoo.collections.Tuple2; * there is no support for recycling it. * </p> * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public class Receiver<T> { /** diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadFactoryFactory.java b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadFactoryFactory.java index 10c057421af..1779179efcd 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadFactoryFactory.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadFactoryFactory.java @@ -19,6 +19,7 @@ public class ThreadFactoryFactory { } return p.getFactory(false); } + static public synchronized ThreadFactory getDaemonThreadFactory(String name) { PooledFactory p = factory.get(name); if (p == null) { @@ -27,6 +28,7 @@ public class ThreadFactoryFactory { } return p.getFactory(true); } + private static class PooledFactory { private static class Factory implements ThreadFactory { final ThreadGroup group; @@ -65,6 +67,6 @@ public class ThreadFactoryFactory { private final String name; private final AtomicInteger poolId = new AtomicInteger(1); } - static private Map<String, PooledFactory> factory = new HashMap<>(); + static private final Map<String, PooledFactory> factory = new HashMap<>(); } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadRobustList.java b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadRobustList.java index 18fec46a888..619e2beb6f4 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/ThreadRobustList.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/ThreadRobustList.java @@ -14,8 +14,7 @@ import java.util.NoSuchElementException; * This is useful for traced information as there may be timed out threads * working on the structure after it is returned upwards for consumption. * - * @since 4.2 - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen * @author bratseth */ public class ThreadRobustList<T> implements Iterable<T> { diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLock.java b/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLock.java index fc43fcc3116..d05cb14c64f 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLock.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLock.java @@ -7,6 +7,7 @@ package com.yahoo.concurrent.classlock; * @author valerijf */ public class ClassLock implements AutoCloseable { + private final Class<?> clazz; private final ClassLocking classLocking; @@ -24,4 +25,5 @@ public class ClassLock implements AutoCloseable { public void close() { classLocking.unlock(clazz, this); } + } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLocking.java b/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLocking.java index f06ef662de5..7c9403965ac 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLocking.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/classlock/ClassLocking.java @@ -12,6 +12,7 @@ import java.util.function.BooleanSupplier; * @author valerijf */ public class ClassLocking { + private final Map<String, ClassLock> classLocks = new HashMap<>(); private final Object monitor = new Object(); @@ -71,4 +72,5 @@ public class ClassLocking { monitor.notifyAll(); } } + } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/classlock/LockInterruptException.java b/vespajlib/src/main/java/com/yahoo/concurrent/classlock/LockInterruptException.java index ea61e87cd80..0e9ece62883 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/classlock/LockInterruptException.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/classlock/LockInterruptException.java @@ -4,6 +4,5 @@ package com.yahoo.concurrent.classlock; /** * @author valerijf */ -@SuppressWarnings("serial") public class LockInterruptException extends RuntimeException { } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 9f27ac507c9..da22dbdc336 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -113,7 +113,9 @@ public abstract class Maintainer implements Runnable { successFactor = maintain(); } catch (UncheckedTimeoutException e) { - if ( ! ignoreCollision) + if (ignoreCollision) + successFactor = 1; + else log.log(Level.WARNING, this + " collided with another run. Will retry in " + interval); } catch (Throwable e) { diff --git a/vespajlib/src/main/java/com/yahoo/data/access/ArrayTraverser.java b/vespajlib/src/main/java/com/yahoo/data/access/ArrayTraverser.java index 9d86e7f02c9..5b22f55273b 100644 --- a/vespajlib/src/main/java/com/yahoo/data/access/ArrayTraverser.java +++ b/vespajlib/src/main/java/com/yahoo/data/access/ArrayTraverser.java @@ -5,12 +5,14 @@ package com.yahoo.data.access; * Callback interface for traversing arrays. * Implement this and call Inspector.traverse() * and you will get one callback for each array entry. - **/ + */ public interface ArrayTraverser { + /** * Callback function to implement. * @param idx array index for the current array entry. * @param inspector accessor for the current array entry's value. - **/ - public void entry(int idx, Inspector inspector); + */ + void entry(int idx, Inspector inspector); + } diff --git a/vespajlib/src/main/java/com/yahoo/data/access/Inspectable.java b/vespajlib/src/main/java/com/yahoo/data/access/Inspectable.java index 362c6e8f69e..9ff173ece45 100644 --- a/vespajlib/src/main/java/com/yahoo/data/access/Inspectable.java +++ b/vespajlib/src/main/java/com/yahoo/data/access/Inspectable.java @@ -5,7 +5,7 @@ package com.yahoo.data.access; * Minimal API to implement for objects containing or exposing * structured, generic, schemaless data. Use this when it's * impractical to implement the Inspector interface directly. - **/ + */ public interface Inspectable { /** Returns an Inspector exposing this object's structured data. */ diff --git a/vespajlib/src/main/java/com/yahoo/data/access/ObjectTraverser.java b/vespajlib/src/main/java/com/yahoo/data/access/ObjectTraverser.java index 45f1d280b34..1d033ebd7b1 100644 --- a/vespajlib/src/main/java/com/yahoo/data/access/ObjectTraverser.java +++ b/vespajlib/src/main/java/com/yahoo/data/access/ObjectTraverser.java @@ -5,12 +5,14 @@ package com.yahoo.data.access; * Callback interface for traversing objects. * Implement this and call Inspector.traverse() * and you will get one callback for each field in an object. - **/ + */ public interface ObjectTraverser { + /** * Callback function to implement. * @param name the name of the current field. * @param inspector accessor for the current field's value. - **/ - public void field(String name, Inspector inspector); + */ + void field(String name, Inspector inspector); + } diff --git a/vespajlib/src/main/java/com/yahoo/data/access/Type.java b/vespajlib/src/main/java/com/yahoo/data/access/Type.java index f0c3f91f386..354568e8808 100644 --- a/vespajlib/src/main/java/com/yahoo/data/access/Type.java +++ b/vespajlib/src/main/java/com/yahoo/data/access/Type.java @@ -13,7 +13,9 @@ package com.yahoo.data.access; * as an array of bytes. * - maps should be represented as an ARRAY of OBJECTs where each * object has the fields "key" and "value". - **/ + */ public enum Type { + EMPTY, BOOL, LONG, DOUBLE, STRING, DATA, ARRAY, OBJECT; + } diff --git a/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java b/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java index f2a0a05e6d3..cfc95c4e05f 100644 --- a/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java +++ b/vespajlib/src/main/java/com/yahoo/data/access/simple/JsonRender.java @@ -32,7 +32,7 @@ public final class JsonRender { private final StringBuilder out; private boolean head = true; - private boolean compact; + private final boolean compact; private int level = 0; public StringEncoder(StringBuilder out, boolean compact) { diff --git a/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java b/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java index 182865bd950..2f4c8c59f1d 100644 --- a/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java +++ b/vespajlib/src/main/java/com/yahoo/data/access/simple/Value.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.data.access.simple; - import com.yahoo.data.access.*; import java.util.Collections; import java.util.Map; @@ -10,11 +9,11 @@ import java.util.List; import java.util.ArrayList; import java.nio.charset.StandardCharsets; - public class Value implements Inspector { - private static Value empty = new EmptyValue(); - private static Value invalid = new Value(); - private static byte[] empty_array = new byte[0]; + + private static final Value empty = new EmptyValue(); + private static final Value invalid = new Value(); + private static final byte[] empty_array = new byte[0]; public static Inspector empty() { return empty; } public static Inspector invalid() { return invalid; } public Inspector inspect() { return this; } diff --git a/vespajlib/src/main/java/com/yahoo/data/access/slime/SlimeAdapter.java b/vespajlib/src/main/java/com/yahoo/data/access/slime/SlimeAdapter.java index 3dc7e80fd05..010a015f288 100644 --- a/vespajlib/src/main/java/com/yahoo/data/access/slime/SlimeAdapter.java +++ b/vespajlib/src/main/java/com/yahoo/data/access/slime/SlimeAdapter.java @@ -1,26 +1,37 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.data.access.slime; +import com.yahoo.slime.Type; import java.util.Map; import java.util.AbstractMap; import java.util.List; import java.util.ArrayList; - public final class SlimeAdapter implements com.yahoo.data.access.Inspector { - private com.yahoo.slime.Inspector inspector; + + private final com.yahoo.slime.Inspector inspector; + public SlimeAdapter(com.yahoo.slime.Inspector inspector) { this.inspector = inspector; } - @Override public boolean equals(Object rhs) { + + @Override + public boolean equals(Object rhs) { if (!(rhs instanceof SlimeAdapter)) { return false; } return inspector.equals(((SlimeAdapter)rhs).inspector); } - @Override public int hashCode() { return inspector.hashCode(); } - @Override public String toString() { return inspector.toString(); } + + @Override + public int hashCode() { return inspector.hashCode(); } + + @Override + public String toString() { return inspector.toString(); } + public com.yahoo.data.access.Inspector inspect() { return this; } + public boolean valid() { return inspector.valid(); } + public com.yahoo.data.access.Type type() { switch(inspector.type()) { case NIX: return com.yahoo.data.access.Type.EMPTY; @@ -34,123 +45,143 @@ public final class SlimeAdapter implements com.yahoo.data.access.Inspector { } return com.yahoo.data.access.Type.EMPTY; } - private boolean verify(com.yahoo.slime.Type ok_type_a) { - com.yahoo.slime.Type my_type = inspector.type(); - return (valid() && (my_type == ok_type_a)); + + private boolean verify(Type okTypeA) { + Type myType = inspector.type(); + return (valid() && (myType == okTypeA)); } - private boolean verify(com.yahoo.slime.Type ok_type_a, - com.yahoo.slime.Type ok_type_b) - { - com.yahoo.slime.Type my_type = inspector.type(); - return (valid() && (my_type == ok_type_a || my_type == ok_type_b)); + + private boolean verify(Type okTypeA, Type okTypeB) { + Type myType = inspector.type(); + return (valid() && (myType == okTypeA || myType == okTypeB)); } - private boolean verify(com.yahoo.slime.Type ok_type_a, - com.yahoo.slime.Type ok_type_b, - com.yahoo.slime.Type ok_type_c) - { - com.yahoo.slime.Type my_type = inspector.type(); - return (valid() && (my_type == ok_type_a || my_type == ok_type_b || my_type == ok_type_c)); + + private boolean verify(Type okTypeA, Type okTypeB, Type okTypeC) { + Type myType = inspector.type(); + return (valid() && (myType == okTypeA || myType == okTypeB || myType == okTypeC)); } + public int entryCount() { return inspector.entries(); } + public int fieldCount() { return inspector.fields(); } + public boolean asBool() { - if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.BOOL)) { + if (!verify(Type.NIX, Type.BOOL)) { throw new IllegalStateException("invalid data extraction!"); } return inspector.asBool(); } + public long asLong() { - if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.LONG, com.yahoo.slime.Type.DOUBLE)) { + if (!verify(Type.NIX, Type.LONG, Type.DOUBLE)) { throw new IllegalStateException("invalid data extraction!"); } return inspector.asLong(); } + public double asDouble() { - if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.DOUBLE, com.yahoo.slime.Type.LONG)) { + if (!verify(Type.NIX, Type.DOUBLE, Type.LONG)) { throw new IllegalStateException("invalid data extraction!"); } return inspector.asDouble(); } + public String asString() { - if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.STRING)) { + if (!verify(Type.NIX, Type.STRING)) { throw new IllegalStateException("invalid data extraction!"); } return inspector.asString(); } + public byte[] asUtf8() { - if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.STRING)) { + if (!verify(Type.NIX, Type.STRING)) { throw new IllegalStateException("invalid data extraction!"); } return inspector.asUtf8(); } + public byte[] asData() { - if (!verify(com.yahoo.slime.Type.NIX, com.yahoo.slime.Type.DATA)) { + if (!verify(Type.NIX, Type.DATA)) { throw new IllegalStateException("invalid data extraction!"); } return inspector.asData(); } + public boolean asBool(boolean defaultValue) { - if (!verify(com.yahoo.slime.Type.BOOL)) { + if (!verify(Type.BOOL)) { return defaultValue; } return inspector.asBool(); } + public long asLong(long defaultValue) { - if (!verify(com.yahoo.slime.Type.LONG, com.yahoo.slime.Type.DOUBLE)) { + if (!verify(Type.LONG, Type.DOUBLE)) { return defaultValue; } return inspector.asLong(); } + public double asDouble(double defaultValue) { - if (!verify(com.yahoo.slime.Type.DOUBLE, com.yahoo.slime.Type.LONG)) { + if (!verify(Type.DOUBLE, Type.LONG)) { return defaultValue; } return inspector.asDouble(); } + public String asString(String defaultValue) { - if (!verify(com.yahoo.slime.Type.STRING)) { + if (!verify(Type.STRING)) { return defaultValue; } return inspector.asString(); } + public byte[] asUtf8(byte[] defaultValue) { - if (!verify(com.yahoo.slime.Type.STRING)) { + if (!verify(Type.STRING)) { return defaultValue; } return inspector.asUtf8(); } + public byte[] asData(byte[] defaultValue) { - if (!verify(com.yahoo.slime.Type.DATA)) { + if (!verify(Type.DATA)) { return defaultValue; } return inspector.asData(); } + public void traverse(final com.yahoo.data.access.ArrayTraverser at) { inspector.traverse(new com.yahoo.slime.ArrayTraverser() { public void entry(int idx, com.yahoo.slime.Inspector inspector) { at.entry(idx, new SlimeAdapter(inspector)); } }); } + public void traverse(final com.yahoo.data.access.ObjectTraverser ot) { inspector.traverse(new com.yahoo.slime.ObjectTraverser() { public void field(String name, com.yahoo.slime.Inspector inspector) { ot.field(name, new SlimeAdapter(inspector)); } }); } + public com.yahoo.data.access.Inspector entry(int idx) { return new SlimeAdapter(inspector.entry(idx)); } + public com.yahoo.data.access.Inspector field(String name) { return new SlimeAdapter(inspector.field(name)); } + public Iterable<com.yahoo.data.access.Inspector> entries() { final List<com.yahoo.data.access.Inspector> list = new ArrayList<>(); + inspector.traverse(new com.yahoo.slime.ArrayTraverser() { public void entry(int idx, com.yahoo.slime.Inspector inspector) { list.add(new SlimeAdapter(inspector)); } }); return list; } + public Iterable<Map.Entry<String,com.yahoo.data.access.Inspector>> fields() { final List<Map.Entry<String,com.yahoo.data.access.Inspector>> list = new ArrayList<>(); inspector.traverse(new com.yahoo.slime.ObjectTraverser() { public void field(String name, com.yahoo.slime.Inspector inspector) { - list.add(new AbstractMap.SimpleImmutableEntry<String,com.yahoo.data.access.Inspector>(name, new SlimeAdapter(inspector))); + list.add(new AbstractMap.SimpleImmutableEntry<>(name, new SlimeAdapter(inspector))); } }); return list; } + } diff --git a/vespajlib/src/main/java/com/yahoo/exception/ExceptionUtils.java b/vespajlib/src/main/java/com/yahoo/exception/ExceptionUtils.java index fbd7336f558..8911e5c5d9e 100644 --- a/vespajlib/src/main/java/com/yahoo/exception/ExceptionUtils.java +++ b/vespajlib/src/main/java/com/yahoo/exception/ExceptionUtils.java @@ -37,4 +37,5 @@ public class ExceptionUtils { throw new UncheckedIOException(e); } } + } diff --git a/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java b/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java index d686a564d99..7a5e02cf30c 100644 --- a/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java +++ b/vespajlib/src/main/java/com/yahoo/geo/BoundingBoxParser.java @@ -40,7 +40,7 @@ public class BoundingBoxParser { /** * parse the given string as a bounding box and return a parser object with parsed coordinates in member variables * @throws IllegalArgumentException if the input is malformed in any way - **/ + */ public BoundingBoxParser(String bb) { this.parseString = bb; this.len = bb.length(); diff --git a/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java b/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java index 986baf82bd2..187807d9a04 100644 --- a/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java +++ b/vespajlib/src/main/java/com/yahoo/geo/DegreesParser.java @@ -5,15 +5,13 @@ package com.yahoo.geo; * Utility for parsing geographical coordinates * * @author arnej27959 - **/ + */ public class DegreesParser { - /** - * the parsed latitude (degrees north if positive) - **/ + + /** The parsed latitude (degrees north if positive). */ public double latitude = 0; - /** - * the parsed longitude (degrees east if positive) - **/ + + /** The parsed longitude (degrees east if positive). */ public double longitude = 0; private boolean isDigit(char ch) { @@ -23,8 +21,8 @@ public class DegreesParser { return (ch == 'N' || ch == 'S' || ch == 'E' || ch == 'W'); } - private String parseString = null; - private int len = 0; + private final String parseString; + private final int len; private int pos = 0; private char getNextChar() throws IllegalArgumentException { @@ -65,9 +63,9 @@ public class DegreesParser { * "E10o25.982;N63o25.105" → same <br> * "N63.418417;E10.433033" → same <br> * "63N25.105;10E25.982" → same <br> - * @param latandlong Latitude and longitude separated by semicolon. * - **/ + * @param latandlong Latitude and longitude separated by semicolon. + */ public DegreesParser(String latandlong) throws IllegalArgumentException { this.parseString = latandlong; this.len = parseString.length(); @@ -107,11 +105,7 @@ public class DegreesParser { if (isDigit(ch) || ch == '.') { valid = true; - if (foundDigits) { - throw new IllegalArgumentException("found digits after not consuming previous digits"); - } double divider = 1.0; - foundDot = false; while (isDigit(ch)) { foundDigits = true; accum *= 10; @@ -163,7 +157,6 @@ public class DegreesParser { // if we found some number, assign it into the next unset variable if (foundDigits) { - valid = true; if (degSet) { if (minSet) { if (secSet) { @@ -281,4 +274,5 @@ public class DegreesParser { } // everything parsed OK } + } diff --git a/vespajlib/src/main/java/com/yahoo/geo/DistanceParser.java b/vespajlib/src/main/java/com/yahoo/geo/DistanceParser.java index acb7ed95597..04c2e735055 100644 --- a/vespajlib/src/main/java/com/yahoo/geo/DistanceParser.java +++ b/vespajlib/src/main/java/com/yahoo/geo/DistanceParser.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.geo; import com.yahoo.api.annotations.Beta; /** * Utility for parsing a geographical distance with unit. - **/ + */ @Beta public class DistanceParser { + // according to wikipedia: // Earth's equatorial radius = 6378137 meter - not used // meters per mile = 1609.344 diff --git a/vespajlib/src/main/java/com/yahoo/geo/OneDegreeParser.java b/vespajlib/src/main/java/com/yahoo/geo/OneDegreeParser.java index 5b14606bb39..1ad5842173d 100644 --- a/vespajlib/src/main/java/com/yahoo/geo/OneDegreeParser.java +++ b/vespajlib/src/main/java/com/yahoo/geo/OneDegreeParser.java @@ -6,17 +6,14 @@ package com.yahoo.geo; * Utility for parsing one geographical coordinate * * @author arnej27959 - **/ + */ class OneDegreeParser { - /** - * the parsed latitude (degrees north if positive) - **/ + + /** The parsed latitude (degrees north if positive). */ public double latitude = 0; public boolean foundLatitude = false; - /** - * the parsed longitude (degrees east if positive) - **/ + /** The parsed longitude (degrees east if positive). */ public double longitude = 0; public boolean foundLongitude = false; @@ -78,6 +75,7 @@ class OneDegreeParser { * "E10o25.982" and "N63o25.105" → same <br> * "N63.418417" and "E10.433033" → same <br> * "63N25.105" and "10E25.982" → same <br> + * * @param assumeNorthSouth Latitude assumed, otherwise longitude * @param toParse Latitude or longitude string to parse * @@ -130,9 +128,6 @@ class OneDegreeParser { if (isDigit(ch) || ch == '.') { valid = true; - if (foundDigits) { - throw new IllegalArgumentException("found digits after not consuming previous digits when parsing <"+parseString+">"); - } double divider = 1.0; foundDot = false; while (isDigit(ch)) { @@ -279,4 +274,5 @@ class OneDegreeParser { } throw new IllegalArgumentException("found neither latitude nor longitude from: "+parseString); } + } diff --git a/vespajlib/src/main/java/com/yahoo/geo/ParsedDegree.java b/vespajlib/src/main/java/com/yahoo/geo/ParsedDegree.java index 695d78eadfc..d17508206d8 100644 --- a/vespajlib/src/main/java/com/yahoo/geo/ParsedDegree.java +++ b/vespajlib/src/main/java/com/yahoo/geo/ParsedDegree.java @@ -6,13 +6,14 @@ package com.yahoo.geo; * Utility for holding one geographical coordinate * * @author arnej27959 - **/ + */ public class ParsedDegree { + /** * the parsed latitude or longitude * Degrees north or east if positive * Degrees south or west if negative - **/ + */ public final double degrees; // one of these two flag will be true: @@ -46,6 +47,7 @@ public class ParsedDegree { throw new IllegalArgumentException("could not parse: "+toParse); } + @Override public String toString() { if (isLatitude) { return "Latitude: "+degrees+" degrees"; diff --git a/vespajlib/src/main/java/com/yahoo/geo/ZCurve.java b/vespajlib/src/main/java/com/yahoo/geo/ZCurve.java index d53ab72b199..45865cd265d 100644 --- a/vespajlib/src/main/java/com/yahoo/geo/ZCurve.java +++ b/vespajlib/src/main/java/com/yahoo/geo/ZCurve.java @@ -8,6 +8,7 @@ package com.yahoo.geo; * @author gjoranv */ public class ZCurve { + /** * Encode two 32 bit integers by bit-interleaving them into one 64 bit * integer value. The x-direction owns the least significant bit (bit @@ -43,8 +44,8 @@ public class ZCurve { * @return The bit-interleaved long containing x and y. */ public static long encode(int x, int y) { - long xl = (long)x; - long yl = (long)y; + long xl = x; + long yl = y; long rx = ((xl & 0x00000000ffff0000L) << 16) | (xl & 0x000000000000ffffL); long ry = ((yl & 0x00000000ffff0000L) << 16) | (yl & 0x000000000000ffffL); @@ -120,7 +121,7 @@ public class ZCurve { * * @param x x value * @param y y value - * @return The bit-interleaved long containing x and y. + * @return the bit-interleaved long containing x and y */ public static long encode_slow(int x, int y) { long z = 0L; diff --git a/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java b/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java index e97abe4a80a..a7cb6a9508e 100644 --- a/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java +++ b/vespajlib/src/main/java/com/yahoo/io/AbstractByteWriter.java @@ -16,8 +16,7 @@ import java.nio.charset.CharsetEncoder; * @author Steinar Knutsen * @author baldersheim */ -public abstract class AbstractByteWriter extends GenericWriter implements - WritableByteTransmitter { +public abstract class AbstractByteWriter extends GenericWriter implements WritableByteTransmitter { protected final CharsetEncoder encoder; protected final BufferChain buffer; diff --git a/vespajlib/src/main/java/com/yahoo/io/BufferChain.java b/vespajlib/src/main/java/com/yahoo/io/BufferChain.java index c3dec43764b..cee7a3c25dd 100644 --- a/vespajlib/src/main/java/com/yahoo/io/BufferChain.java +++ b/vespajlib/src/main/java/com/yahoo/io/BufferChain.java @@ -18,6 +18,7 @@ import java.util.List; * @author Steinar Knutsen */ public final class BufferChain { + // refer to the revision history of ByteWriter for more information about // the reasons behind the sizing of BUFFERSIZE, WATERMARK and MAXBUFFERS static final int BUFFERSIZE = 4096; diff --git a/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java b/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java index df457990bbe..4bca8ad51f7 100644 --- a/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java +++ b/vespajlib/src/main/java/com/yahoo/io/ByteWriter.java @@ -14,6 +14,7 @@ import java.nio.charset.CharsetEncoder; * @author Steinar Knutsen */ public class ByteWriter extends AbstractByteWriter { + private final OutputStream stream; public ByteWriter(final OutputStream stream, final CharsetEncoder encoder) { diff --git a/vespajlib/src/main/java/com/yahoo/io/FatalErrorHandler.java b/vespajlib/src/main/java/com/yahoo/io/FatalErrorHandler.java index a491d7ee2cb..1aa84b0a15c 100644 --- a/vespajlib/src/main/java/com/yahoo/io/FatalErrorHandler.java +++ b/vespajlib/src/main/java/com/yahoo/io/FatalErrorHandler.java @@ -28,8 +28,8 @@ import java.util.logging.Level; * * @author Steinar Knutsen */ - public class FatalErrorHandler { + protected static final Logger log = Logger.getLogger(FatalErrorHandler.class.getName()); /** @@ -48,4 +48,5 @@ public class FatalErrorHandler { Runtime.getRuntime().halt(1); } } + } diff --git a/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java b/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java index ef598f70175..4b7d2ba4094 100644 --- a/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java +++ b/vespajlib/src/main/java/com/yahoo/io/GrowableBufferOutputStream.java @@ -15,12 +15,12 @@ import java.nio.ByteBuffer; public class GrowableBufferOutputStream extends OutputStream { private ByteBuffer lastBuffer; - private ByteBuffer directBuffer; - private LinkedList<ByteBuffer> bufferList = new LinkedList<>(); - private Stack<ByteBuffer> recycledBuffers = new Stack<>(); + private final ByteBuffer directBuffer; + private final LinkedList<ByteBuffer> bufferList = new LinkedList<>(); + private final Stack<ByteBuffer> recycledBuffers = new Stack<>(); - private int bufferSize; - private int maxBuffers; + private final int bufferSize; + private final int maxBuffers; public GrowableBufferOutputStream(int bufferSize, int maxBuffers) { this.bufferSize = bufferSize; diff --git a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java index 54cdf5c7b40..81e1306b29e 100644 --- a/vespajlib/src/main/java/com/yahoo/io/IOUtils.java +++ b/vespajlib/src/main/java/com/yahoo/io/IOUtils.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.io; - import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; @@ -27,7 +26,6 @@ import java.util.List; import java.nio.charset.Charset; import java.nio.ByteBuffer; - /** * Some static io convenience methods. * diff --git a/vespajlib/src/main/java/com/yahoo/io/Utf8ByteWriter.java b/vespajlib/src/main/java/com/yahoo/io/Utf8ByteWriter.java index c4cf25f34a0..30ad4aefd09 100644 --- a/vespajlib/src/main/java/com/yahoo/io/Utf8ByteWriter.java +++ b/vespajlib/src/main/java/com/yahoo/io/Utf8ByteWriter.java @@ -7,11 +7,13 @@ import java.io.IOException; import java.nio.ByteBuffer; public class Utf8ByteWriter extends AbstractByteWriter { + private ByteBuffer myBuf; public Utf8ByteWriter(int initialBuffer) { super(Utf8.getNewEncoder()); myBuf = ByteBuffer.allocate(initialBuffer); } + @Override public void send(ByteBuffer src) throws IOException { if (myBuf.remaining() < src.remaining()) { @@ -36,6 +38,7 @@ public class Utf8ByteWriter extends AbstractByteWriter { /** * Return a buffer ready for read. Must only be called after writer has been closed. + * * @return A flipped ByteBuffer */ public ByteBuffer getBuf() { diff --git a/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java b/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java index ed0b1c05d4d..128b564d2eb 100644 --- a/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java +++ b/vespajlib/src/main/java/com/yahoo/protect/ClassValidator.java @@ -16,7 +16,7 @@ import java.util.List; * is minimal. * </p> * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ public final class ClassValidator { @@ -25,10 +25,8 @@ public final class ClassValidator { * maskedClass is implemented in testClass. Note, this will by definition * blow up on final methods in maskedClass. * - * @param testClass - * class which wraps or masks another class - * @param maskedClass - * class which is masked or wrapped + * @param testClass class which wraps or masks another class + * @param maskedClass class which is masked or wrapped * @return the methods which seem to miss from testClass to be complete */ public static List<Method> unmaskedMethods(Class<?> testClass, @@ -53,8 +51,7 @@ public final class ClassValidator { * Check testClass overrides all protected, public and package private * methods of its immediate super class. See unmaskedMethods(). * - * @param testClass - * the class to check whether completely masks its super class + * @param testClass the class to check whether completely masks its super class * @return the methods missing from testClass to completely override its * immediate super class */ diff --git a/vespajlib/src/main/java/com/yahoo/protect/Process.java b/vespajlib/src/main/java/com/yahoo/protect/Process.java index 1a462f1042e..8caaec88117 100644 --- a/vespajlib/src/main/java/com/yahoo/protect/Process.java +++ b/vespajlib/src/main/java/com/yahoo/protect/Process.java @@ -11,7 +11,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import java.util.logging.Logger; - /** * A class for interacting with the global state of the running VM. * diff --git a/vespajlib/src/main/java/com/yahoo/protect/Validator.java b/vespajlib/src/main/java/com/yahoo/protect/Validator.java index 358d75408da..530404cbd52 100644 --- a/vespajlib/src/main/java/com/yahoo/protect/Validator.java +++ b/vespajlib/src/main/java/com/yahoo/protect/Validator.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.protect; - /** * Static utility methods for validating input. * diff --git a/vespajlib/src/main/java/com/yahoo/reflection/Casting.java b/vespajlib/src/main/java/com/yahoo/reflection/Casting.java index ea11980b385..7d904543760 100644 --- a/vespajlib/src/main/java/com/yahoo/reflection/Casting.java +++ b/vespajlib/src/main/java/com/yahoo/reflection/Casting.java @@ -5,12 +5,15 @@ import java.util.Optional; /** * Utility methods for doing casting + * * @author Tony Vaagenes */ public class Casting { + /** * Returns the casted instance if it is assignment-compatible with targetClass, * or empty otherwise. + * * @see Class#isInstance(Object) */ public static <T> Optional<T> cast(Class<T> targetClass, Object instance) { @@ -18,4 +21,5 @@ public class Casting { Optional.of(targetClass.cast(instance)): Optional.empty(); } + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/ArrayInserter.java b/vespajlib/src/main/java/com/yahoo/slime/ArrayInserter.java index 993556922d0..e2bb7479574 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/ArrayInserter.java +++ b/vespajlib/src/main/java/com/yahoo/slime/ArrayInserter.java @@ -4,24 +4,26 @@ package com.yahoo.slime; /** * Helper class for inserting values into an ArrayValue. * For justification read Inserter documentation. - **/ + */ public final class ArrayInserter implements Inserter { + private Cursor target; public ArrayInserter(Cursor c) { target = c; } - public final ArrayInserter adjust(Cursor c) { + public ArrayInserter adjust(Cursor c) { target = c; return this; } - public final Cursor insertNIX() { return target.addNix(); } - public final Cursor insertBOOL(boolean value) { return target.addBool(value); } - public final Cursor insertLONG(long value) { return target.addLong(value); } - public final Cursor insertDOUBLE(double value) { return target.addDouble(value); } - public final Cursor insertSTRING(String value) { return target.addString(value); } - public final Cursor insertSTRING(byte[] utf8) { return target.addString(utf8); } - public final Cursor insertDATA(byte[] value) { return target.addData(value); } - public final Cursor insertARRAY() { return target.addArray(); } - public final Cursor insertOBJECT() { return target.addObject(); } + public Cursor insertNIX() { return target.addNix(); } + public Cursor insertBOOL(boolean value) { return target.addBool(value); } + public Cursor insertLONG(long value) { return target.addLong(value); } + public Cursor insertDOUBLE(double value) { return target.addDouble(value); } + public Cursor insertSTRING(String value) { return target.addString(value); } + public Cursor insertSTRING(byte[] utf8) { return target.addString(utf8); } + public Cursor insertDATA(byte[] value) { return target.addData(value); } + public Cursor insertARRAY() { return target.addArray(); } + public Cursor insertOBJECT() { return target.addObject(); } + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java index 383fd626a62..b9aa8e5cf22 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryDecoder.java @@ -8,6 +8,7 @@ import static com.yahoo.slime.BinaryFormat.decode_type; import static com.yahoo.slime.BinaryFormat.decode_zigzag; final class BinaryDecoder { + BufferedInput in; private final SlimeInserter slimeInserter = new SlimeInserter(null); @@ -160,4 +161,5 @@ final class BinaryDecoder { } } } + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java index 0d848e83bab..7da85b5cb63 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java @@ -5,9 +5,8 @@ import static com.yahoo.slime.BinaryFormat.encode_double; import static com.yahoo.slime.BinaryFormat.encode_type_and_meta; import static com.yahoo.slime.BinaryFormat.encode_zigzag; -final class BinaryEncoder implements -ArrayTraverser, ObjectSymbolTraverser -{ +final class BinaryEncoder implements ArrayTraverser, ObjectSymbolTraverser { + private final BufferedOutput out; BinaryEncoder() { @@ -143,4 +142,5 @@ ArrayTraverser, ObjectSymbolTraverser encode_cmpr_long(symbol); encodeValue(inspector); } + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java index b397d4c2982..519ab54e25d 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryFormat.java @@ -6,8 +6,9 @@ import com.yahoo.compress.Compressor; /** * Class for serializing Slime data into binary format, or deserializing * the binary format into a Slime object. - **/ + */ public class BinaryFormat { + static long encode_zigzag(long x) { return ((x << 1) ^ (x >> 63)); // note ASR } @@ -96,4 +97,5 @@ public class BinaryFormat { BinaryDecoder decoder = new BinaryDecoder(); return decoder.decode(data, offset, length); } + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/Injector.java b/vespajlib/src/main/java/com/yahoo/slime/Injector.java index eec5f3999f9..71b5574fcc8 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Injector.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Injector.java @@ -5,6 +5,7 @@ package com.yahoo.slime; * @author hakonhall */ public class Injector { + /** * Inject a slime sub-structure described by an Inspector into a slime * structure where the insertion point is described by an @@ -17,7 +18,7 @@ public class Injector { * * @param inspector what to inject * @param inserter where to inject - **/ + */ public void inject(Inspector inspector, Inserter inserter) { if (inspector.valid()) { injectValue(inserter, inspector, null); @@ -77,4 +78,5 @@ public class Injector { injectValue(new ObjectInserter(cursor, name), inspector, guard); } } + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/Inserter.java b/vespajlib/src/main/java/com/yahoo/slime/Inserter.java index 7a7e5ba6f84..c36b56c11e2 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Inserter.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Inserter.java @@ -6,8 +6,9 @@ package com.yahoo.slime; * classes (ArrayValue, ObjectValue, or Slime). May be useful for * deserializers where you can use it to decouple the actual value * decoding from the container where the value should be inserted. - **/ + */ public interface Inserter { + Cursor insertNIX(); Cursor insertBOOL(boolean value); Cursor insertLONG(long value); @@ -17,4 +18,5 @@ public interface Inserter { Cursor insertDATA(byte[] value); Cursor insertARRAY(); Cursor insertOBJECT(); + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java index e7a96695922..3772aafa044 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/slime/JsonFormat.java @@ -14,8 +14,8 @@ import java.io.*; * * @author Ulf Lilleengen */ -public final class JsonFormat implements SlimeFormat -{ +public final class JsonFormat implements SlimeFormat { + private final static byte [] HEX = Utf8.toBytes("0123456789ABCDEF"); private final boolean compact; public JsonFormat(boolean compact) { diff --git a/vespajlib/src/main/java/com/yahoo/slime/ObjectInserter.java b/vespajlib/src/main/java/com/yahoo/slime/ObjectInserter.java index c6becab554e..57c99f4604e 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/ObjectInserter.java +++ b/vespajlib/src/main/java/com/yahoo/slime/ObjectInserter.java @@ -5,6 +5,7 @@ package com.yahoo.slime; * @author hakonhall */ public final class ObjectInserter implements Inserter { + private Cursor target; private String key; diff --git a/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolInserter.java b/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolInserter.java index c5c18943926..4935d96d388 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolInserter.java +++ b/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolInserter.java @@ -4,8 +4,9 @@ package com.yahoo.slime; /** * Helper class for inserting values into an ObjectValue. * For justification read Inserter documentation. - **/ + */ public final class ObjectSymbolInserter implements Inserter { + private Cursor target; private int symbol; @@ -14,19 +15,19 @@ public final class ObjectSymbolInserter implements Inserter { symbol = sym; } - public final ObjectSymbolInserter adjust(Cursor c, int sym) { + public ObjectSymbolInserter adjust(Cursor c, int sym) { target = c; symbol = sym; return this; } - public final Cursor insertNIX() { return target.setNix(symbol); } - public final Cursor insertBOOL(boolean value) { return target.setBool(symbol, value); } - public final Cursor insertLONG(long value) { return target.setLong(symbol, value); } - public final Cursor insertDOUBLE(double value) { return target.setDouble(symbol, value); } - public final Cursor insertSTRING(String value) { return target.setString(symbol, value); } - public final Cursor insertSTRING(byte[] utf8) { return target.setString(symbol, utf8); } - public final Cursor insertDATA(byte[] value) { return target.setData(symbol, value); } - public final Cursor insertARRAY() { return target.setArray(symbol); } - public final Cursor insertOBJECT() { return target.setObject(symbol); } + public Cursor insertNIX() { return target.setNix(symbol); } + public Cursor insertBOOL(boolean value) { return target.setBool(symbol, value); } + public Cursor insertLONG(long value) { return target.setLong(symbol, value); } + public Cursor insertDOUBLE(double value) { return target.setDouble(symbol, value); } + public Cursor insertSTRING(String value) { return target.setString(symbol, value); } + public Cursor insertSTRING(byte[] utf8) { return target.setString(symbol, utf8); } + public Cursor insertDATA(byte[] value) { return target.setData(symbol, value); } + public Cursor insertARRAY() { return target.setArray(symbol); } + public Cursor insertOBJECT() { return target.setObject(symbol); } } diff --git a/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolTraverser.java b/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolTraverser.java index 90ca9bc70cd..a4946d3ff31 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolTraverser.java +++ b/vespajlib/src/main/java/com/yahoo/slime/ObjectSymbolTraverser.java @@ -6,12 +6,14 @@ package com.yahoo.slime; * Implement this and call Inspector.traverse() * and you will get one callback for each field in an object. **/ -public interface ObjectSymbolTraverser -{ +public interface ObjectSymbolTraverser { + /** * Callback function to implement. + * * @param sym symbol id for the current field. * @param inspector accessor for the current field's value. - **/ - public void field(int sym, Inspector inspector); + */ + void field(int sym, Inspector inspector); + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/ObjectTraverser.java b/vespajlib/src/main/java/com/yahoo/slime/ObjectTraverser.java index 6597d3b82f3..d9f9b75dfd1 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/ObjectTraverser.java +++ b/vespajlib/src/main/java/com/yahoo/slime/ObjectTraverser.java @@ -7,6 +7,7 @@ package com.yahoo.slime; * and you will get one callback for each field in an object. */ public interface ObjectTraverser { + /** * Callback function to implement. * diff --git a/vespajlib/src/main/java/com/yahoo/slime/Slime.java b/vespajlib/src/main/java/com/yahoo/slime/Slime.java index c45b48cf743..eba9226c8ef 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Slime.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Slime.java @@ -7,7 +7,7 @@ package com.yahoo.slime; * ObjectValue data objects). * * @author havardpe - **/ + */ public final class Slime { private final SymbolTable names = new SymbolTable(); diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeStream.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeStream.java index 4fea24110f0..0c0229579e7 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/SlimeStream.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeStream.java @@ -11,6 +11,7 @@ import java.util.stream.Stream; * @author ogronnesby */ public final class SlimeStream { + private SlimeStream() {} /** @@ -24,4 +25,5 @@ public final class SlimeStream { .mapToObj(array::entry) .map(mapper); } + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/SymbolTable.java b/vespajlib/src/main/java/com/yahoo/slime/SymbolTable.java index 0e498aa72e7..e86835fc0b7 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/SymbolTable.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SymbolTable.java @@ -5,7 +5,7 @@ package com.yahoo.slime; * A mapping from an arbitrary set of unique strings to a range of * integers. Slime users normally won't need to use this class * directly. - **/ + */ final class SymbolTable { public static final int INVALID = Integer.MAX_VALUE; @@ -95,4 +95,5 @@ final class SymbolTable { } return INVALID; } + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/Type.java b/vespajlib/src/main/java/com/yahoo/slime/Type.java index 7d2b309157e..5407b296b38 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Type.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Type.java @@ -3,7 +3,7 @@ package com.yahoo.slime; /** * Enumeration of all possibly Slime data types. - **/ + */ public enum Type { NIX(0), diff --git a/vespajlib/src/main/java/com/yahoo/slime/Utf8Codec.java b/vespajlib/src/main/java/com/yahoo/slime/Utf8Codec.java index 73be9620c22..e9f11a90af2 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Utf8Codec.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Utf8Codec.java @@ -5,12 +5,14 @@ import com.yahoo.text.Utf8; /** * Helper class for conversion between String and UTF-8 representations. - **/ + */ class Utf8Codec { + public static String decode(byte[] data, int pos, int len) { return Utf8.toString(data, pos, len); } public static byte[] encode(String str) { return Utf8.toBytes(str); } + } diff --git a/vespajlib/src/main/java/com/yahoo/slime/Visitor.java b/vespajlib/src/main/java/com/yahoo/slime/Visitor.java index 5372e9cbf6f..0e49d0daf1e 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/Visitor.java +++ b/vespajlib/src/main/java/com/yahoo/slime/Visitor.java @@ -4,19 +4,20 @@ package com.yahoo.slime; /** * Visitor interface used to resolve the underlying type of a value * represented by an Inspector. - **/ + */ public interface Visitor { + /** * Called when the visited Inspector is not valid. - **/ - public void visitInvalid(); - public void visitNix(); - public void visitBool(boolean bit); - public void visitLong(long l); - public void visitDouble(double d); - public void visitString(String str); - public void visitString(byte[] utf8); - public void visitData(byte[] data); - public void visitArray(Inspector arr); - public void visitObject(Inspector obj); + */ + void visitInvalid(); + void visitNix(); + void visitBool(boolean bit); + void visitLong(long l); + void visitDouble(double d); + void visitString(String str); + void visitString(byte[] utf8); + void visitData(byte[] data); + void visitArray(Inspector arr); + void visitObject(Inspector obj); } diff --git a/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java b/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java index 0689e634cf4..aedd7047567 100644 --- a/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java +++ b/vespajlib/src/main/java/com/yahoo/system/CommandLineParser.java @@ -11,23 +11,24 @@ import java.util.*; * progname -binaryswitch foo -unaryswitch argument1 argument2 * * @author vegardh - * */ public class CommandLineParser { - private List<String> inputStrings = new ArrayList<>(); - private Map<String, String> legalUnarySwitches = new HashMap<>(); - private Map<String, String> legalBinarySwitches = new HashMap<>(); - private List<String> unarySwitches = new ArrayList<>(); - private Map<String, String> binarySwitches = new HashMap<>(); - private List<String> arguments = new ArrayList<>(); - private Map<String, String> requiredUnarySwitches = new HashMap<>(); - private Map<String, String> requiredBinarySwitches = new HashMap<>(); + + private static final HashSet<String> helpSwitches = new HashSet<>(); + + private final List<String> inputStrings; + private final Map<String, String> legalUnarySwitches = new HashMap<>(); + private final Map<String, String> legalBinarySwitches = new HashMap<>(); + private final List<String> unarySwitches = new ArrayList<>(); + private final Map<String, String> binarySwitches = new HashMap<>(); + private final List<String> arguments = new ArrayList<>(); + private final Map<String, String> requiredUnarySwitches = new HashMap<>(); + private final Map<String, String> requiredBinarySwitches = new HashMap<>(); private String progname = "progname"; private String argumentExplanation; - private int minArguments=0; - private int maxArguments=Integer.MAX_VALUE; + private int minArguments = 0; + private int maxArguments = Integer.MAX_VALUE; private String helpText; - private static HashSet<String> helpSwitches = new HashSet<>(); private boolean helpSwitchUsed = false; static { @@ -213,4 +214,5 @@ public class CommandLineParser { public boolean helpSwitchUsed() { return helpSwitchUsed; } + } diff --git a/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java b/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java index d627a4324d9..6869fdfaf5c 100644 --- a/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java +++ b/vespajlib/src/main/java/com/yahoo/system/ForceLoad.java @@ -3,21 +3,17 @@ package com.yahoo.system; /** * Utility class used to force the loading of other classes. - **/ + */ public class ForceLoad { /** * Force the loading of the given classes. If any of the named * classes can not be loaded, an error will be thrown. * - * @param packageName the name of the package for which - * we want to forceload classes. - * @param classNames array of names of classes (without package prefix) - * to force load. - **/ - public static void forceLoad(String packageName, String[] classNames, ClassLoader loader) - throws ForceLoadError - { + * @param packageName the name of the package for which we want to forceload classes. + * @param classNames array of names of classes (without package prefix) to force load. + */ + public static void forceLoad(String packageName, String[] classNames, ClassLoader loader) throws ForceLoadError { String fullClassName = ""; try { for (String className : classNames) { @@ -28,4 +24,5 @@ public class ForceLoad { throw new ForceLoadError(fullClassName, e); } } + } diff --git a/vespajlib/src/main/java/com/yahoo/system/ForceLoadError.java b/vespajlib/src/main/java/com/yahoo/system/ForceLoadError.java index b85fb1b303c..02fe11e5f5e 100644 --- a/vespajlib/src/main/java/com/yahoo/system/ForceLoadError.java +++ b/vespajlib/src/main/java/com/yahoo/system/ForceLoadError.java @@ -3,8 +3,7 @@ package com.yahoo.system; /** * Special error to be propagated when force-loading a class fails. - **/ -@SuppressWarnings("serial") + */ public class ForceLoadError extends java.lang.Error { /** @@ -12,8 +11,9 @@ public class ForceLoadError extends java.lang.Error { * * @param className full name of offending class * @param cause what caused the failure - **/ + */ public ForceLoadError(String className, Throwable cause) { super("Force loading class '" + className + "' failed", cause); } + } diff --git a/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java index b705cc081a5..91c3e59fbae 100644 --- a/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java +++ b/vespajlib/src/main/java/com/yahoo/system/ProcessExecuter.java @@ -27,7 +27,7 @@ public class ProcessExecuter { /** * Executes the given command synchronously without timeout. * - * @return Retcode and stdout/stderr merged + * @return retcode and stdout/stderr merged */ public Pair<Integer, String> exec(String command) throws IOException { StringTokenizer tok = new StringTokenizer(command); diff --git a/vespajlib/src/main/java/com/yahoo/text/AbstractUtf8Array.java b/vespajlib/src/main/java/com/yahoo/text/AbstractUtf8Array.java index 4994479047e..62998621181 100644 --- a/vespajlib/src/main/java/com/yahoo/text/AbstractUtf8Array.java +++ b/vespajlib/src/main/java/com/yahoo/text/AbstractUtf8Array.java @@ -5,45 +5,32 @@ import java.nio.ByteBuffer; /** * @author baldersheim - * @since 5.2 */ public abstract class AbstractUtf8Array implements Comparable<AbstractUtf8Array> { - /** - * This will write the utf8 sequence to the given target. - */ + + /** Writes the utf8 sequence to the given target. */ final public void writeTo(ByteBuffer target) { target.put(getBytes(), getByteOffset(), getByteLength()); } - /** - * This will return the byte at the given position. - */ + /** Returns the byte at the given position. */ public byte getByte(int index) { return getBytes()[getByteOffset() + index]; } - /** - * - * @return Length in bytes of the utf8 sequence. - */ + /** Returns the length in bytes of the utf8 sequence. */ public abstract int getByteLength(); - /** - * Wraps the utf8 sequence in a ByteBuffer - * @return The wrapping buffer. + /** Wraps the utf8 sequence in a ByteBuffer + * + * @return the wrapping buffer */ public ByteBuffer wrap() { return ByteBuffer.wrap(getBytes(), getByteOffset(), getByteLength()); } - /** - * - * @return The backing byte array. - */ + /** Returns the backing byte array. */ protected abstract byte [] getBytes(); public boolean isEmpty() { return getByteLength() == 0; } - /** - * - * @return The offset in the backing array where the utf8 sequence starts. - */ + /** Returns the offset in the backing array where the utf8 sequence starts. */ protected abstract int getByteOffset(); @Override public int hashCode() { @@ -57,6 +44,7 @@ public abstract class AbstractUtf8Array implements Comparable<AbstractUtf8Array> } return h; } + @Override public boolean equals(Object o) { if (o instanceof AbstractUtf8Array) { @@ -68,10 +56,7 @@ public abstract class AbstractUtf8Array implements Comparable<AbstractUtf8Array> return false; } - /** - * Will convert the utf8 sequence to a Java string - * @return The converted Java String - */ + /** Retuerns the utf8 sequence as a Java string. */ @Override public String toString() { return Utf8.toString(getBytes(), getByteOffset(), getByteLength()); diff --git a/vespajlib/src/main/java/com/yahoo/text/Ascii7BitMatcher.java b/vespajlib/src/main/java/com/yahoo/text/Ascii7BitMatcher.java index d796d746f37..85ecbaf140e 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Ascii7BitMatcher.java +++ b/vespajlib/src/main/java/com/yahoo/text/Ascii7BitMatcher.java @@ -7,9 +7,11 @@ import java.util.BitSet; * Fast replacement for regex based validators of simple expressions. * It can take a list of legal characters for the the first character, * and another list for the following. Limited to 7bit ascii. + * * @author baldersheim */ public class Ascii7BitMatcher { + private final BitSet legalFirst; private final BitSet legalRest; private static BitSet createBitSet(String legal) { @@ -48,7 +50,7 @@ public class Ascii7BitMatcher { return true; } static public String charsAndNumbers() { - char [] chars = new char[26*2+10]; + char[] chars = new char[26*2+10]; int i = 0; for (char c = 'A'; c <= 'Z'; c++) { chars[i++] = c; @@ -61,4 +63,5 @@ public class Ascii7BitMatcher { } return new String(chars); } + } diff --git a/vespajlib/src/main/java/com/yahoo/text/CaseInsensitiveIdentifier.java b/vespajlib/src/main/java/com/yahoo/text/CaseInsensitiveIdentifier.java index d0c2a27cdee..d42db0250ec 100644 --- a/vespajlib/src/main/java/com/yahoo/text/CaseInsensitiveIdentifier.java +++ b/vespajlib/src/main/java/com/yahoo/text/CaseInsensitiveIdentifier.java @@ -5,6 +5,7 @@ package com.yahoo.text; * @author baldersheim */ public class CaseInsensitiveIdentifier extends Identifier { + private final Identifier original; public CaseInsensitiveIdentifier(String s) { @@ -18,4 +19,5 @@ public class CaseInsensitiveIdentifier extends Identifier { original = new Identifier(utf8); } public String toString() { return original.toString(); } + } diff --git a/vespajlib/src/main/java/com/yahoo/text/DataTypeIdentifier.java b/vespajlib/src/main/java/com/yahoo/text/DataTypeIdentifier.java index d6d0cbca624..3f64993c343 100644 --- a/vespajlib/src/main/java/com/yahoo/text/DataTypeIdentifier.java +++ b/vespajlib/src/main/java/com/yahoo/text/DataTypeIdentifier.java @@ -5,6 +5,7 @@ package com.yahoo.text; * @author baldersheim */ public class DataTypeIdentifier { + private static final byte [] ARRAY = {'a', 'r', 'r', 'a', 'y'}; private static final byte [] ANNOTATIONREFERENCE = {'a','n','n','o','t','a','t','i','o','n','r','e','f','e','r','e','n','c','e'}; private static final byte [] MAP = { 'm', 'a', 'p'}; diff --git a/vespajlib/src/main/java/com/yahoo/text/ForwardWriter.java b/vespajlib/src/main/java/com/yahoo/text/ForwardWriter.java index 3fb2e37649c..aedf474359b 100644 --- a/vespajlib/src/main/java/com/yahoo/text/ForwardWriter.java +++ b/vespajlib/src/main/java/com/yahoo/text/ForwardWriter.java @@ -4,10 +4,9 @@ package com.yahoo.text; import java.io.IOException; /** - * Wraps another writer and also converting IOException to Exceptions. + * Wraps another writer and converts IOException to RuntimeExceptions. * * @author baldersheim - * @since 5.2 */ public class ForwardWriter extends GenericWriter { diff --git a/vespajlib/src/main/java/com/yahoo/text/GenericWriter.java b/vespajlib/src/main/java/com/yahoo/text/GenericWriter.java index 29f0f2d4fc5..cc867c00ee2 100644 --- a/vespajlib/src/main/java/com/yahoo/text/GenericWriter.java +++ b/vespajlib/src/main/java/com/yahoo/text/GenericWriter.java @@ -9,19 +9,13 @@ import java.io.Writer; * java.io.Writer, but it allows for more overrides for speed. * This introduces additional interfaces in addition to the java.lang.Writer. * The purpose is to allow for optimizations. + * * @author baldersheim - * @since 5.2 */ - public abstract class GenericWriter extends Writer { -/* - public abstract void write(char [] c, int offset, int bytes); - public abstract void flush(); - public abstract void close(); -*/ public GenericWriter write(char c) throws java.io.IOException { - char t[] = new char[1]; + char[] t = new char[1]; t[0] = c; try { write(t, 0, 1); @@ -47,18 +41,22 @@ public abstract class GenericWriter extends Writer { write(String.valueOf(i)); return this; } + public GenericWriter write(byte i) throws java.io.IOException { write(String.valueOf(i)); return this; } + public GenericWriter write(double i) throws java.io.IOException { write(String.valueOf(i)); return this; } + public GenericWriter write(float i) throws java.io.IOException { write(String.valueOf(i)); return this; } + public GenericWriter write(boolean i) throws java.io.IOException { write(String.valueOf(i)); return this; @@ -68,4 +66,5 @@ public abstract class GenericWriter extends Writer { write(v.toString()); return this; } + } diff --git a/vespajlib/src/main/java/com/yahoo/text/HTML.java b/vespajlib/src/main/java/com/yahoo/text/HTML.java index c7295695414..a983df4d970 100644 --- a/vespajlib/src/main/java/com/yahoo/text/HTML.java +++ b/vespajlib/src/main/java/com/yahoo/text/HTML.java @@ -1,17 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; - import java.util.Map; import java.util.HashMap; - /** * Static HTML escaping stuff * - * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + * @author Bjorn Borud */ public class HTML { + static Object[][] entities = { // {"#39", Integer.valueOf(39)}, // ' - apostrophe {"quot", 34}, // " - double-quote @@ -121,4 +120,5 @@ public class HTML { } return buf.toString(); } + } diff --git a/vespajlib/src/main/java/com/yahoo/text/JSONWriter.java b/vespajlib/src/main/java/com/yahoo/text/JSONWriter.java index bf89d9c4547..30746cf016c 100644 --- a/vespajlib/src/main/java/com/yahoo/text/JSONWriter.java +++ b/vespajlib/src/main/java/com/yahoo/text/JSONWriter.java @@ -16,7 +16,7 @@ import java.util.Deque; public final class JSONWriter { /** A stack maintaining the "needs comma" state at the current level */ - private Deque<Boolean> needsComma=new ArrayDeque<>(); + private final Deque<Boolean> needsComma = new ArrayDeque<>(); private static final char[] DIGITS = new char[] { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; diff --git a/vespajlib/src/main/java/com/yahoo/text/JavaWriterWriter.java b/vespajlib/src/main/java/com/yahoo/text/JavaWriterWriter.java index cfbcf7353d3..1edc1c06400 100644 --- a/vespajlib/src/main/java/com/yahoo/text/JavaWriterWriter.java +++ b/vespajlib/src/main/java/com/yahoo/text/JavaWriterWriter.java @@ -8,7 +8,6 @@ import java.io.Writer; * Wraps a simple java.lang.Writer. Of course you loose the possible optimizations. * * @author baldersheim - * @since 5.2 */ public final class JavaWriterWriter extends GenericWriter { @@ -44,6 +43,7 @@ public final class JavaWriterWriter extends GenericWriter { throw new RuntimeException("Caught exception in Java writer.close.", e); } } - public final Writer getWriter() { return out; } + + public Writer getWriter() { return out; } } diff --git a/vespajlib/src/main/java/com/yahoo/text/LowercaseIdentifier.java b/vespajlib/src/main/java/com/yahoo/text/LowercaseIdentifier.java index 0091fed2a67..9b2d57df5eb 100644 --- a/vespajlib/src/main/java/com/yahoo/text/LowercaseIdentifier.java +++ b/vespajlib/src/main/java/com/yahoo/text/LowercaseIdentifier.java @@ -5,6 +5,7 @@ package com.yahoo.text; * @author baldersheim */ public class LowercaseIdentifier extends Identifier { + public LowercaseIdentifier(String s) { this(Utf8.toBytes(s)); } diff --git a/vespajlib/src/main/java/com/yahoo/text/MapParser.java b/vespajlib/src/main/java/com/yahoo/text/MapParser.java index cd0ff19afd1..9b40e3d90ad 100644 --- a/vespajlib/src/main/java/com/yahoo/text/MapParser.java +++ b/vespajlib/src/main/java/com/yahoo/text/MapParser.java @@ -24,7 +24,6 @@ import java.util.Map; * <p>Map parsers are NOT multithread safe, but are cheap to construct.</p> * * @author bratseth - * @since 5.1.15 */ public abstract class MapParser<VALUETYPE> extends SimpleMapParser { diff --git a/vespajlib/src/main/java/com/yahoo/text/PositionedString.java b/vespajlib/src/main/java/com/yahoo/text/PositionedString.java index 9285f8ba8af..aca0b7d1259 100644 --- a/vespajlib/src/main/java/com/yahoo/text/PositionedString.java +++ b/vespajlib/src/main/java/com/yahoo/text/PositionedString.java @@ -6,11 +6,10 @@ package com.yahoo.text; * Useful for writing simple single-pass parsers. * * @author bratseth - * @since 5.1.15 */ public class PositionedString { - private String s; + private final String s; private int p; /** diff --git a/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java b/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java index 185f2e34ab8..6724ae51a84 100644 --- a/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java +++ b/vespajlib/src/main/java/com/yahoo/text/SimpleMapParser.java @@ -24,7 +24,6 @@ package com.yahoo.text; * <p>Map parsers are NOT multithread safe, but are cheap to construct.</p> * * @author bratseth - * @since 5.1.15 */ public abstract class SimpleMapParser { diff --git a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java index 0b6d707e0fd..fb90fabb964 100644 --- a/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java +++ b/vespajlib/src/main/java/com/yahoo/text/StringUtilities.java @@ -4,6 +4,7 @@ package com.yahoo.text; import com.google.common.collect.ImmutableSet; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; import java.io.ByteArrayOutputStream; @@ -19,14 +20,15 @@ import java.util.Set; // TODO: Text utilities should which are still needed should move to Text. This should be deprecated. public class StringUtilities { - private static Charset UTF8 = Charset.forName("utf8"); + private static final Charset UTF8 = StandardCharsets.UTF_8; private static byte toHex(int val) { return (byte) (val < 10 ? '0' + val : 'a' + (val - 10)); } private static class ReplacementCharacters { - public byte needEscape[] = new byte[256]; - public byte replacement1[] = new byte[256]; - public byte replacement2[] = new byte[256]; + + public byte[] needEscape = new byte[256]; + public byte[] replacement1 = new byte[256]; + public byte[] replacement2 = new byte[256]; public ReplacementCharacters() { for (int i=0; i<256; ++i) { @@ -65,7 +67,7 @@ public class StringUtilities { * @return The escaped string */ public static String escape(String source, char delimiter) { - byte bytes[] = source.getBytes(UTF8); + byte[] bytes = source.getBytes(UTF8); ByteArrayOutputStream result = new ByteArrayOutputStream(); for (byte b : bytes) { int val = b; @@ -90,7 +92,7 @@ public class StringUtilities { } public static String unescape(String source) { - byte bytes[] = source.getBytes(UTF8); + byte[] bytes = source.getBytes(UTF8); ByteArrayOutputStream result = new ByteArrayOutputStream(); for (int i=0; i<bytes.length; ++i) { if (bytes[i] != '\\') { @@ -212,7 +214,7 @@ public class StringUtilities { public static Set<String> split(String s) { if (s == null || s.isEmpty()) return Collections.emptySet(); ImmutableSet.Builder<String> b = new ImmutableSet.Builder<>(); - for (String item : s.split("[\\s\\,]")) + for (String item : s.split("[\\s,]")) if ( ! item.isEmpty()) b.add(item); return b.build(); diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8.java b/vespajlib/src/main/java/com/yahoo/text/Utf8.java index a71406582fa..91bbb86d3fc 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Utf8.java +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8.java @@ -18,7 +18,6 @@ import java.nio.charset.StandardCharsets; * @author arnej27959 * @author Steinar Knutsen * @author baldersheim - * */ public final class Utf8 { diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8Array.java b/vespajlib/src/main/java/com/yahoo/text/Utf8Array.java index 75498ecc69c..2d881ea2f62 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Utf8Array.java +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8Array.java @@ -1,18 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; - import java.nio.ByteBuffer; /** * This is a primitive class that owns an array of utf8 encoded string. * This is a class that has speed as its primary purpose. * If you have a string, consider Utf8String - * If you have a large backing array consider Utf8PartialArray + * If you have a large backing array consider Utf8PartialArray. + * * @author baldersheim - * @since 5.2 */ - public class Utf8Array extends AbstractUtf8Array { protected final byte[] utf8; diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8PartialArray.java b/vespajlib/src/main/java/com/yahoo/text/Utf8PartialArray.java index 7f76d17ce8e..275335d3c2b 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Utf8PartialArray.java +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8PartialArray.java @@ -3,10 +3,11 @@ package com.yahoo.text; /** * This wraps a window in a backing byte array. Without doing any copying. + * * @author baldersheim - * @since 5.2 */ public class Utf8PartialArray extends Utf8Array { + final int offset; final int length; diff --git a/vespajlib/src/main/java/com/yahoo/text/Utf8String.java b/vespajlib/src/main/java/com/yahoo/text/Utf8String.java index 1530f7effff..b36f2933291 100644 --- a/vespajlib/src/main/java/com/yahoo/text/Utf8String.java +++ b/vespajlib/src/main/java/com/yahoo/text/Utf8String.java @@ -1,18 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.text; - /** * String with Utf8 backing. + * * @author baldersheim - * @since 5.2 */ -public final class Utf8String extends Utf8Array implements CharSequence -{ +public final class Utf8String extends Utf8Array implements CharSequence { + private final String s; /** * This will construct a utf8 backing of the given string. + * * @param str The string that will be utf8 encoded */ public Utf8String(String str) { @@ -22,6 +22,7 @@ public final class Utf8String extends Utf8Array implements CharSequence /** * This will create a string based on the utf8 sequence. + * * @param utf8 The backing array */ public Utf8String(AbstractUtf8Array utf8) { diff --git a/vespajlib/src/main/java/com/yahoo/time/TimeBudget.java b/vespajlib/src/main/java/com/yahoo/time/TimeBudget.java index cf69cfb4fcd..b18a0f397f6 100644 --- a/vespajlib/src/main/java/com/yahoo/time/TimeBudget.java +++ b/vespajlib/src/main/java/com/yahoo/time/TimeBudget.java @@ -14,6 +14,7 @@ import java.util.Optional; * @author hakon */ public class TimeBudget { + private final Clock clock; private final Instant start; private final Optional<Duration> timeout; @@ -90,4 +91,5 @@ public class TimeBudget { private static Duration makeNonNegative(Duration duration) { return duration.isNegative() ? Duration.ZERO : duration; } + } diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/BufferSerializer.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/BufferSerializer.java index 1bf71f51112..716795049fb 100644 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/BufferSerializer.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/BufferSerializer.java @@ -11,6 +11,7 @@ import java.nio.ByteOrder; * @author baldersheim */ public class BufferSerializer implements Serializer, Deserializer { + protected GrowableByteBuffer buf; public BufferSerializer(GrowableByteBuffer buf) { this.buf = buf; } diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Deserializer.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Deserializer.java index d8ee9308146..affc20c73b9 100644 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/Deserializer.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Deserializer.java @@ -5,6 +5,7 @@ package com.yahoo.vespa.objects; * @author baldersheim */ public interface Deserializer { + byte getByte(FieldBase field); short getShort(FieldBase field); int getInt(FieldBase field); @@ -13,4 +14,5 @@ public interface Deserializer { double getDouble(FieldBase field); byte [] getBytes(FieldBase field, int length); String getString(FieldBase field); + } diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java index 947b312ac3b..5fc6336f628 100644 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Identifiable.java @@ -27,7 +27,7 @@ public class Identifiable extends Selectable implements Cloneable { * Returns the class identifier of this class. This proxies the {@link #onGetClassId()} method that must be * implemented by every subclass. * - * @return The class identifier. + * @return the class identifier */ public final int getClassId() { return onGetClassId(); diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectOperation.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectOperation.java index 1eff5c0120b..f78ff98cc0a 100755 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectOperation.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectOperation.java @@ -13,5 +13,6 @@ public interface ObjectOperation { * * @param obj The object to operate on. */ - public void execute(Object obj); + void execute(Object obj); + } diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectPredicate.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectPredicate.java index a555c7688ae..6da1f6d6d7d 100755 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectPredicate.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/ObjectPredicate.java @@ -15,5 +15,6 @@ public interface ObjectPredicate { * @param obj The object to check. * @return True or false. */ - public boolean check(Object obj); + boolean check(Object obj); + } diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Serializer.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Serializer.java index 4dc27392caa..570ba97f24f 100644 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/Serializer.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Serializer.java @@ -7,6 +7,7 @@ import java.nio.ByteBuffer; * @author baldersheim */ public interface Serializer { + Serializer putByte(FieldBase field, byte value); Serializer putShort(FieldBase field, short value); Serializer putInt(FieldBase field, int value); @@ -16,4 +17,5 @@ public interface Serializer { Serializer put(FieldBase field, byte[] value); Serializer put(FieldBase field, ByteBuffer value); Serializer put(FieldBase field, String value); + } diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 9f98d23d6a9..eb7e1f7d4c0 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -31,6 +31,7 @@ vespa_define_module( src/tests/component src/tests/compress src/tests/compression + src/tests/cpu_usage src/tests/crypto src/tests/data/databuffer src/tests/data/input_reader diff --git a/vespalib/src/tests/cpu_usage/CMakeLists.txt b/vespalib/src/tests/cpu_usage/CMakeLists.txt new file mode 100644 index 00000000000..e3e2def6056 --- /dev/null +++ b/vespalib/src/tests/cpu_usage/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_cpu_usage_test_app TEST + SOURCES + cpu_usage_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_cpu_usage_test_app COMMAND vespalib_cpu_usage_test_app) diff --git a/vespalib/src/tests/cpu_usage/cpu_usage_test.cpp b/vespalib/src/tests/cpu_usage/cpu_usage_test.cpp new file mode 100644 index 00000000000..98a7bd780a7 --- /dev/null +++ b/vespalib/src/tests/cpu_usage/cpu_usage_test.cpp @@ -0,0 +1,103 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/cpu_usage.h> +#include <vespa/vespalib/util/benchmark_timer.h> +#include <vespa/vespalib/testkit/test_kit.h> + +#include <thread> + +using namespace vespalib; + +bool verbose = false; +size_t loop_cnt = 10; +double budget = 0.25; + +using Sampler = vespalib::cpu_usage::ThreadSampler; + +//----------------------------------------------------------------------------- + +void be_busy(duration d) { + if (d > 0ms) { + volatile int tmp = 123; + auto t0 = steady_clock::now(); + while ((steady_clock::now() - t0) < d) { + for (int i = 0; i < 1000; ++i) { + tmp = (tmp + i); + tmp = (tmp - i); + } + } + } +} + +std::vector<duration> sample(const std::vector<Sampler*> &list) { + std::vector<duration> result; + result.reserve(list.size()); + for (auto *sampler: list) { + result.push_back(sampler->sample()); + } + return result; +} + +//----------------------------------------------------------------------------- + +void verify_sampling(size_t thread_id, size_t num_threads, std::vector<Sampler*> &samplers, bool force_mock) { + if (thread_id == 0) { + TEST_BARRIER(); // #1 + auto t0 = steady_clock::now(); + std::vector<duration> pre_usage = sample(samplers); + TEST_BARRIER(); // #2 + TEST_BARRIER(); // #3 + auto t1 = steady_clock::now(); + std::vector<duration> post_usage = sample(samplers); + TEST_BARRIER(); // #4 + double wall = to_s(t1 - t0); + std::vector<double> load(4, 0.0); + for (size_t i = 0; i < 4; ++i) { + load[i] = to_s(post_usage[i] - pre_usage[i]) / wall; + } + EXPECT_GREATER(load[3], load[0]); + fprintf(stderr, "loads: { %.2f, %.2f, %.2f, %.2f }\n", load[0], load[1], load[2], load[3]); + } else { + int idx = (thread_id - 1); + double target_load = double(thread_id - 1) / (num_threads - 2); + auto sampler = cpu_usage::create_thread_sampler(force_mock, target_load); + samplers[idx] = sampler.get(); + TEST_BARRIER(); // #1 + TEST_BARRIER(); // #2 + for (size_t i = 0; i < loop_cnt; ++i) { + be_busy(std::chrono::milliseconds(idx)); + } + TEST_BARRIER(); // #3 + TEST_BARRIER(); // #4 + } +} + +//----------------------------------------------------------------------------- + +TEST_MT_F("require that dummy thread-based CPU usage sampling with known expected load works", 5, std::vector<Sampler*>(4, nullptr)) { + TEST_DO(verify_sampling(thread_id, num_threads, f1, true)); +} + +TEST_MT_F("require that external thread-based CPU usage sampling works", 5, std::vector<Sampler*>(4, nullptr)) { + TEST_DO(verify_sampling(thread_id, num_threads, f1, false)); +} + +TEST("measure thread CPU clock overhead") { + auto sampler = cpu_usage::create_thread_sampler(); + duration d; + double min_time_us = BenchmarkTimer::benchmark([&d, &sampler]() noexcept { d = sampler->sample(); }, budget) * 1000000.0; + fprintf(stderr, "approx overhead per sample (thread CPU clock): %f us\n", min_time_us); +} + +//----------------------------------------------------------------------------- + +int main(int argc, char **argv) { + TEST_MASTER.init(__FILE__); + if ((argc == 2) && (argv[1] == std::string("verbose"))) { + verbose = true; + loop_cnt = 1000; + budget = 5.0; + } + TEST_RUN_ALL(); + return (TEST_MASTER.fini() ? 0 : 1); +} diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 58f6a93babc..32f679d22d7 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -18,6 +18,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT classname.cpp compress.cpp compressor.cpp + cpu_usage.cpp destructor_callbacks.cpp dual_merge_director.cpp error.cpp diff --git a/vespalib/src/vespa/vespalib/util/cpu_usage.cpp b/vespalib/src/vespa/vespalib/util/cpu_usage.cpp new file mode 100644 index 00000000000..4eee0a63870 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/cpu_usage.cpp @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "cpu_usage.h" +#include "require.h" +#include <pthread.h> + +namespace vespalib { + +namespace cpu_usage { + +namespace { + +class DummyThreadSampler : public ThreadSampler { +private: + steady_time _start; + double _load; +public: + DummyThreadSampler(double load) : _start(steady_clock::now()), _load(load) {} + duration sample() const override { + return from_s(to_s(steady_clock::now() - _start) * _load); + } +}; + +#ifdef __linux__ + +class LinuxThreadSampler : public ThreadSampler { +private: + clockid_t _my_clock; +public: + LinuxThreadSampler() : _my_clock() { + REQUIRE_EQ(pthread_getcpuclockid(pthread_self(), &_my_clock), 0); + } + duration sample() const override { + timespec ts; + REQUIRE_EQ(clock_gettime(_my_clock, &ts), 0); + return from_timespec(ts); + } +}; + +#endif + +} // <unnamed> + +ThreadSampler::UP create_thread_sampler(bool force_mock_impl, double expected_load) { + if (force_mock_impl) { + return std::make_unique<DummyThreadSampler>(expected_load); + } +#ifdef __linux__ + return std::make_unique<LinuxThreadSampler>(); +#endif + return std::make_unique<DummyThreadSampler>(expected_load); +} + +} // cpu_usage + +} // namespace diff --git a/vespalib/src/vespa/vespalib/util/cpu_usage.h b/vespalib/src/vespa/vespalib/util/cpu_usage.h new file mode 100644 index 00000000000..09509a984b5 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/cpu_usage.h @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/time.h> +#include <memory> + +namespace vespalib { + +namespace cpu_usage { + +/** + * Samples the total CPU usage of the thread that created it. Note + * that this must not be used after thread termination. Enables + * sampling the CPU usage of a thread from outside the thread. + **/ +struct ThreadSampler { + using UP = std::unique_ptr<ThreadSampler>; + virtual duration sample() const = 0; + virtual ~ThreadSampler() {} +}; + +ThreadSampler::UP create_thread_sampler(bool force_mock_impl = false, double expected_load = 0.16); + +} // cpu_usage + +} // namespace diff --git a/vespalib/src/vespa/vespalib/util/time.h b/vespalib/src/vespa/vespalib/util/time.h index f1f529b64b6..f19d71afb32 100644 --- a/vespalib/src/vespa/vespalib/util/time.h +++ b/vespalib/src/vespa/vespalib/util/time.h @@ -67,6 +67,10 @@ constexpr duration from_timeval(const timeval & tv) { return duration(tv.tv_sec*1000000000L + tv.tv_usec*1000L); } +constexpr duration from_timespec(const timespec & ts) { + return duration(ts.tv_sec*1000000000L + ts.tv_nsec); +} + vespalib::string to_string(system_time time); /** diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java index 63f4ade952a..40bfb70109d 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java @@ -1239,25 +1239,25 @@ public class MockCuratorFramework implements CuratorFramework { } @Override - public TransactionCreateBuilder create() { + public TransactionCreateBuilder<CuratorTransactionBridge> create() { ensureNotCommitted(); return new MockTransactionCreateBuilder(); } @Override - public TransactionDeleteBuilder delete() { + public TransactionDeleteBuilder<CuratorTransactionBridge> delete() { ensureNotCommitted(); return new MockTransactionDeleteBuilder(); } @Override - public TransactionSetDataBuilder setData() { + public TransactionSetDataBuilder<CuratorTransactionBridge> setData() { ensureNotCommitted(); return new MockTransactionSetDataBuilder(); } @Override - public TransactionCheckBuilder check() { + public TransactionCheckBuilder<CuratorTransactionBridge> check() { ensureNotCommitted(); throw new UnsupportedOperationException("Not implemented in MockCurator"); } @@ -1266,7 +1266,7 @@ public class MockCuratorFramework implements CuratorFramework { if (committed) throw new IllegalStateException("transaction already committed"); } - private class MockTransactionCreateBuilder implements TransactionCreateBuilder { + private class MockTransactionCreateBuilder implements TransactionCreateBuilder<CuratorTransactionBridge> { private CreateMode createMode = CreateMode.PERSISTENT; @@ -1294,22 +1294,22 @@ public class MockCuratorFramework implements CuratorFramework { } @Override - public TransactionCreateBuilder2 withTtl(long l) { + public TransactionCreateBuilder2<CuratorTransactionBridge> withTtl(long l) { return this; } @Override - public Object withACL(List list, boolean b) { + public MockTransactionCreateBuilder withACL(List<ACL> list, boolean b) { return this; } @Override - public Object withACL(List list) { + public MockTransactionCreateBuilder withACL(List<ACL> list) { return this; } } - private class MockTransactionDeleteBuilder implements TransactionDeleteBuilder { + private class MockTransactionDeleteBuilder implements TransactionDeleteBuilder<CuratorTransactionBridge> { @Override public Pathable<CuratorTransactionBridge> withVersion(int i) { @@ -1324,7 +1324,7 @@ public class MockCuratorFramework implements CuratorFramework { } - private class MockTransactionSetDataBuilder implements TransactionSetDataBuilder { + private class MockTransactionSetDataBuilder implements TransactionSetDataBuilder<CuratorTransactionBridge> { @Override public VersionPathAndBytesable<CuratorTransactionBridge> compressed() { diff --git a/zookeeper-server/zookeeper-server-3.6.3/pom.xml b/zookeeper-server/zookeeper-server-3.6.3/pom.xml index f7e6f512f7c..a8ad183de4e 100644 --- a/zookeeper-server/zookeeper-server-3.6.3/pom.xml +++ b/zookeeper-server/zookeeper-server-3.6.3/pom.xml @@ -11,6 +11,9 @@ <artifactId>zookeeper-server-3.6.3</artifactId> <packaging>container-plugin</packaging> <version>7-SNAPSHOT</version> + <properties> + <zookeeper.version>3.6.3</zookeeper.version> + </properties> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -32,7 +35,7 @@ <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> - <version>3.6.3</version> + <version>${zookeeper.version}</version> <exclusions> <!-- Container provides wiring for all common log libraries @@ -87,7 +90,6 @@ <configuration> <compilerArgs> <arg>-Xlint:all</arg> - <arg>-Werror</arg> </compilerArgs> </configuration> </plugin> @@ -97,6 +99,9 @@ <configuration> <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> <forkMode>once</forkMode> + <systemPropertyVariables> + <zk-version>${zookeeper.version}</zk-version> + </systemPropertyVariables> </configuration> </plugin> <plugin> diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java index c002ffa72ce..246911fdfc7 100644 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java @@ -18,21 +18,21 @@ import java.util.concurrent.atomic.AtomicReference; */ public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer { - private final AtomicReference<QuorumPeer> peer = new AtomicReference<>(); + private QuorumPeer peer; @Inject public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) { - reconfigurer.startOrReconfigure(zookeeperServerConfig, this, VespaQuorumPeer::new, peer::set); + peer = reconfigurer.startOrReconfigure(zookeeperServerConfig, this, () -> peer = new VespaQuorumPeer()); } @Override public void shutdown() { - peer.get().shutdown(Duration.ofMinutes(1)); + peer.shutdown(Duration.ofMinutes(1)); } @Override public void start(Path configFilePath) { - peer.get().start(configFilePath); + peer.start(configFilePath); } @Override diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java index 27aa18c64c7..f0a95b70e96 100644 --- a/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java +++ b/zookeeper-server/zookeeper-server-3.6.3/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java @@ -2,11 +2,15 @@ package com.yahoo.vespa.zookeeper; import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; +import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.admin.ZooKeeperAdmin; +import org.apache.zookeeper.data.ACL; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -19,27 +23,28 @@ public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { private static final Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperAdminImpl.class.getName()); @Override - public void reconfigure(String connectionSpec, String joiningServers, String leavingServers) throws ReconfigException { - ZooKeeperAdmin zooKeeperAdmin = null; - try { - zooKeeperAdmin = createAdmin(connectionSpec); + public void reconfigure(String connectionSpec, String servers) throws ReconfigException { + try (ZooKeeperAdmin zooKeeperAdmin = createAdmin(connectionSpec)) { long fromConfig = -1; // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0) - byte[] appliedConfig = zooKeeperAdmin.reconfigure(joiningServers, leavingServers, null, fromConfig, null); + log.log(Level.INFO, "Applying ZooKeeper config: " + servers); + byte[] appliedConfig = zooKeeperAdmin.reconfigure(null, null, servers, fromConfig, null); log.log(Level.INFO, "Applied ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); - } catch (KeeperException e) { - if (retryOn(e)) - throw new ReconfigException(e); - else - throw new RuntimeException(e); - } catch (IOException | InterruptedException e) { + + // Verify by issuing a write operation; this is only accepted once new quorum is obtained. + List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; + String node = zooKeeperAdmin.create("/reconfigure-dummy-node", new byte[0], acl, CreateMode.EPHEMERAL_SEQUENTIAL); + zooKeeperAdmin.delete(node, -1); + + log.log(Level.INFO, "Verified ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); + } + catch ( KeeperException.ReconfigInProgress + | KeeperException.ConnectionLossException + | KeeperException.NewConfigNoQuorum e) { + throw new ReconfigException(e); + } + catch (KeeperException | IOException | InterruptedException e) { throw new RuntimeException(e); - } finally { - if (zooKeeperAdmin != null) { - try { - zooKeeperAdmin.close(); - } catch (InterruptedException e) { /* ignore */} - } } } @@ -48,11 +53,5 @@ public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { (event) -> log.log(Level.INFO, event.toString()), new ZkClientConfigBuilder().toConfig()); } - private static boolean retryOn(KeeperException e) { - return e instanceof KeeperException.ReconfigInProgress || - e instanceof KeeperException.ConnectionLossException || - e instanceof KeeperException.NewConfigNoQuorum; - } - } diff --git a/zookeeper-server/zookeeper-server-3.6.3/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java b/zookeeper-server/zookeeper-server-3.6.3/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java new file mode 100644 index 00000000000..922c389f94a --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.6.3/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java @@ -0,0 +1,258 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeper; + +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.net.HostName; +import com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer; +import com.yahoo.vespa.zookeeper.Reconfigurer; +import com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl; +import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.admin.ZooKeeperAdmin; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.Phaser; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.IntStream; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.stream.Collectors.toList; +import static org.junit.Assert.assertEquals; + +public class VespaZooKeeperTest { + + static final Path tempDirRoot = getTmpDir(); + static final List<Integer> ports = new ArrayList<>(); + + /** + * Performs dynamic reconfiguration of ZooKeeper servers. + * + * First, a cluster of 3 servers is set up, and some data is written to it. + * Then, 3 new servers are added, and the first 3 marked for retirement; + * this should force the quorum to move the 3 new servers, but not disconnect the old ones. + * Next, the old servers are removed. + * Then, the cluster is reduced to size 1. + * Finally, the cluster grows to size 3 again. + * + * Throughout all of this, quorum should remain, and the data should remain the same. + */ + @Test(timeout = 120_000) + @Ignore // Unstable, some ZK server keeps resetting connections sometimes. + public void testReconfiguration() throws ExecutionException, InterruptedException, IOException, KeeperException, TimeoutException { + List<ZooKeeper> keepers = new ArrayList<>(); + for (int i = 0; i < 8; i++) keepers.add(new ZooKeeper()); + for (int i = 0; i < 8; i++) keepers.get(i).run(); + + // Start the first three servers. + List<ZookeeperServerConfig> configs = getConfigs(0, 0, 3, 0); + for (int i = 0; i < 3; i++) keepers.get(i).config = configs.get(i); + for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Wait for all servers to be up and running. + for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Write data to verify later. + String path = writeData(configs.get(0)); + + // Let three new servers join, causing the three older ones to retire and leave the ensemble. + configs = getConfigs(0, 3, 3, 3); + for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); + // The existing servers can't reconfigure and leave before the joiners are up. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Wait for new quorum to be established. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Verify written data is preserved. + verifyData(path, configs.get(3)); + + // Old servers are removed. + configs = getConfigs(3, 0, 3, 0); + for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); + // Old servers shut down, while the newer servers remain. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // Ensure old servers shut down properly. + for (int i = 0; i < 3; i++) keepers.get(i).await(); + // Ensure new servers have reconfigured. + for (int i = 3; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Verify written data is preserved. + verifyData(path, configs.get(3)); + + + // Cluster shrinks to a single server. + configs = getConfigs(5, 0, 1, 0); + for (int i = 3; i < 6; i++) keepers.get(i).config = configs.get(i); + for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // We let the remaining server reconfigure the others out before they die. + for (int i = 3; i < 5; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 3; i < 5; i++) keepers.get(i).await(); + verifyData(path, configs.get(5)); + + // Cluster grows to 3 servers again. + configs = getConfigs(5, 0, 3, 2); + for (int i = 5; i < 8; i++) keepers.get(i).config = configs.get(i); + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // Wait for the joiners. + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + verifyData(path, configs.get(7)); + + // Let the remaining servers terminate. + for (int i = 5; i < 8; i++) keepers.get(i).config = null; + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 5; i < 8; i++) keepers.get(i).await(); + } + + static String writeData(ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { + ZooKeeperAdmin admin = createAdmin(config); + List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; + String node = admin.create("/test-node", "hi".getBytes(UTF_8), acl, CreateMode.EPHEMERAL_SEQUENTIAL); + String read = new String(admin.getData(node, false, new Stat()), UTF_8); + assertEquals("hi", read); + return node; + } + + static void verifyData(String path, ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { + for (int i = 0; i < 10; i++) { + try { + assertEquals("hi", new String(createAdmin(config).getData(path, false, new Stat()), UTF_8)); + return; + } + catch (KeeperException.ConnectionLossException e) { + e.printStackTrace(); + Thread.sleep(10 << i); + } + } + } + + static ZooKeeperAdmin createAdmin(ZookeeperServerConfig config) throws IOException { + return new ZooKeeperAdmin(HostName.getLocalhost() + ":" + config.clientPort(), + 10_000, + System.err::println, + new ZkClientConfigBuilder().toConfig()); + } + + static class ZooKeeper { + + final ExecutorService executor = Executors.newSingleThreadExecutor(); + final Phaser phaser = new Phaser(2); + final AtomicReference<Future<?>> future = new AtomicReference<>(); + ZookeeperServerConfig config; + + void run() { + future.set(executor.submit(() -> { + Reconfigurer reconfigurer = new Reconfigurer(new VespaZooKeeperAdminImpl()); + phaser.arriveAndAwaitAdvance(); + while (config != null) { + new ReconfigurableVespaZooKeeperServer(reconfigurer, config); + phaser.arriveAndAwaitAdvance(); // server is now up, let test thread sync here + phaser.arriveAndAwaitAdvance(); // wait before reconfig/teardown to let test thread do stuff + } + reconfigurer.deconstruct(); + })); + } + + void await() throws ExecutionException, InterruptedException, TimeoutException { + future.get().get(30, SECONDS); + } + } + + static List<ZookeeperServerConfig> getConfigs(int removed, int retired, int active, int joining) { + return IntStream.rangeClosed(1, removed + retired + active) + .mapToObj(id -> getConfig(removed, retired, active, joining, id)) + .collect(toList()); + } + + // Config for server #id among retired + active servers, of which the last may be joining, and with offset removed. + static ZookeeperServerConfig getConfig(int removed, int retired, int active, int joining, int id) { + if (id <= removed) + return null; + + Path tempDir = tempDirRoot.resolve("zookeeper-" + id); + return new ZookeeperServerConfig.Builder() + .clientPort(getPorts(id).get(0)) + .dataDir(tempDir.toString()) + .zooKeeperConfigFile(tempDir.resolve("zookeeper.cfg").toString()) + .myid(id) + .myidFile(tempDir.resolve("myid").toString()) + .dynamicReconfiguration(true) + .server(IntStream.rangeClosed(removed + 1, removed + retired + active) + .mapToObj(i -> new ZookeeperServerConfig.Server.Builder() + .id(i) + .clientPort(getPorts(i).get(0)) + .electionPort(getPorts(i).get(1)) + .quorumPort(getPorts(i).get(2)) + .hostname("localhost") + .joining(i - removed > retired + active - joining) + .retired(i - removed <= retired)) + .collect(toList())) + .build(); + } + + static List<Integer> getPorts(int id) { + if (ports.size() < id * 3) { + int previousPort; + if (ports.isEmpty()) { + String[] version = System.getProperty("zk-version").split("\\."); + int versionPortOffset = 0; + for (String part : version) + versionPortOffset = 32 * (versionPortOffset + Integer.parseInt(part)); + previousPort = 20000 + versionPortOffset % 30000; + } + else + previousPort = ports.get(ports.size() - 1); + + for (int i = 0; i < 3; i++) + ports.add(previousPort = nextPort(previousPort)); + } + return ports.subList(id * 3 - 3, id * 3); + } + + static int nextPort(int previousPort) { + for (int j = 1; j <= 30000; j++) { + int port = (previousPort + j); + while (port > 50000) + port -= 30000; + + try (ServerSocket socket = new ServerSocket(port)) { + return socket.getLocalPort(); + } + catch (IOException e) { + System.err.println("Could not bind port " + port + ": " + e); + } + } + throw new RuntimeException("No free ports"); + } + + static Path getTmpDir() { + try { + Path tempDir = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), "vespa-zk-test"); + tempDir.toFile().deleteOnExit(); + return tempDir.toAbsolutePath(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/zookeeper-server/zookeeper-server-3.7.0/pom.xml b/zookeeper-server/zookeeper-server-3.7.0/pom.xml index ac7db35e6af..01fd83a496b 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/pom.xml +++ b/zookeeper-server/zookeeper-server-3.7.0/pom.xml @@ -11,6 +11,9 @@ <artifactId>zookeeper-server-3.7.0</artifactId> <packaging>container-plugin</packaging> <version>7-SNAPSHOT</version> + <properties> + <zookeeper.version>3.7.0</zookeeper.version> + </properties> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -32,7 +35,7 @@ <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> - <version>3.7.0</version> + <version>${zookeeper.version}</version> <exclusions> <!-- Container provides wiring for all common log libraries @@ -87,7 +90,6 @@ <configuration> <compilerArgs> <arg>-Xlint:all</arg> - <arg>-Werror</arg> </compilerArgs> </configuration> </plugin> @@ -97,6 +99,9 @@ <configuration> <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> <forkMode>once</forkMode> + <systemPropertyVariables> + <zk-version>${zookeeper.version}</zk-version> + </systemPropertyVariables> </configuration> </plugin> <plugin> diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java index c002ffa72ce..246911fdfc7 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java @@ -18,21 +18,21 @@ import java.util.concurrent.atomic.AtomicReference; */ public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer { - private final AtomicReference<QuorumPeer> peer = new AtomicReference<>(); + private QuorumPeer peer; @Inject public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) { - reconfigurer.startOrReconfigure(zookeeperServerConfig, this, VespaQuorumPeer::new, peer::set); + peer = reconfigurer.startOrReconfigure(zookeeperServerConfig, this, () -> peer = new VespaQuorumPeer()); } @Override public void shutdown() { - peer.get().shutdown(Duration.ofMinutes(1)); + peer.shutdown(Duration.ofMinutes(1)); } @Override public void start(Path configFilePath) { - peer.get().start(configFilePath); + peer.start(configFilePath); } @Override diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java index 27aa18c64c7..ae7bf8d84f5 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java +++ b/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java @@ -2,11 +2,15 @@ package com.yahoo.vespa.zookeeper; import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; +import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.admin.ZooKeeperAdmin; +import org.apache.zookeeper.data.ACL; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -19,27 +23,28 @@ public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { private static final Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperAdminImpl.class.getName()); @Override - public void reconfigure(String connectionSpec, String joiningServers, String leavingServers) throws ReconfigException { - ZooKeeperAdmin zooKeeperAdmin = null; - try { - zooKeeperAdmin = createAdmin(connectionSpec); + public void reconfigure(String connectionSpec, String servers) throws ReconfigException { + try (ZooKeeperAdmin zooKeeperAdmin = createAdmin(connectionSpec)) { long fromConfig = -1; - // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0) - byte[] appliedConfig = zooKeeperAdmin.reconfigure(joiningServers, leavingServers, null, fromConfig, null); + // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0). + log.log(Level.INFO, "Applying ZooKeeper config: " + servers); + byte[] appliedConfig = zooKeeperAdmin.reconfigure(null, null, servers, fromConfig, null); log.log(Level.INFO, "Applied ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); - } catch (KeeperException e) { - if (retryOn(e)) - throw new ReconfigException(e); - else - throw new RuntimeException(e); - } catch (IOException | InterruptedException e) { + + // Verify by issuing a write operation; this is only accepted once new quorum is obtained. + List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; + String node = zooKeeperAdmin.create("/reconfigure-dummy-node", new byte[0], acl, CreateMode.EPHEMERAL_SEQUENTIAL); + zooKeeperAdmin.delete(node, -1); + + log.log(Level.INFO, "Verified ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); + } + catch ( KeeperException.ReconfigInProgress + | KeeperException.ConnectionLossException + | KeeperException.NewConfigNoQuorum e) { + throw new ReconfigException(e); + } + catch (KeeperException | IOException | InterruptedException e) { throw new RuntimeException(e); - } finally { - if (zooKeeperAdmin != null) { - try { - zooKeeperAdmin.close(); - } catch (InterruptedException e) { /* ignore */} - } } } @@ -48,11 +53,5 @@ public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { (event) -> log.log(Level.INFO, event.toString()), new ZkClientConfigBuilder().toConfig()); } - private static boolean retryOn(KeeperException e) { - return e instanceof KeeperException.ReconfigInProgress || - e instanceof KeeperException.ConnectionLossException || - e instanceof KeeperException.NewConfigNoQuorum; - } - } diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java b/zookeeper-server/zookeeper-server-3.7.0/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java new file mode 100644 index 00000000000..922c389f94a --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.7.0/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java @@ -0,0 +1,258 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeper; + +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.net.HostName; +import com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer; +import com.yahoo.vespa.zookeeper.Reconfigurer; +import com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl; +import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.admin.ZooKeeperAdmin; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.Phaser; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.IntStream; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.stream.Collectors.toList; +import static org.junit.Assert.assertEquals; + +public class VespaZooKeeperTest { + + static final Path tempDirRoot = getTmpDir(); + static final List<Integer> ports = new ArrayList<>(); + + /** + * Performs dynamic reconfiguration of ZooKeeper servers. + * + * First, a cluster of 3 servers is set up, and some data is written to it. + * Then, 3 new servers are added, and the first 3 marked for retirement; + * this should force the quorum to move the 3 new servers, but not disconnect the old ones. + * Next, the old servers are removed. + * Then, the cluster is reduced to size 1. + * Finally, the cluster grows to size 3 again. + * + * Throughout all of this, quorum should remain, and the data should remain the same. + */ + @Test(timeout = 120_000) + @Ignore // Unstable, some ZK server keeps resetting connections sometimes. + public void testReconfiguration() throws ExecutionException, InterruptedException, IOException, KeeperException, TimeoutException { + List<ZooKeeper> keepers = new ArrayList<>(); + for (int i = 0; i < 8; i++) keepers.add(new ZooKeeper()); + for (int i = 0; i < 8; i++) keepers.get(i).run(); + + // Start the first three servers. + List<ZookeeperServerConfig> configs = getConfigs(0, 0, 3, 0); + for (int i = 0; i < 3; i++) keepers.get(i).config = configs.get(i); + for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Wait for all servers to be up and running. + for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Write data to verify later. + String path = writeData(configs.get(0)); + + // Let three new servers join, causing the three older ones to retire and leave the ensemble. + configs = getConfigs(0, 3, 3, 3); + for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); + // The existing servers can't reconfigure and leave before the joiners are up. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Wait for new quorum to be established. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Verify written data is preserved. + verifyData(path, configs.get(3)); + + // Old servers are removed. + configs = getConfigs(3, 0, 3, 0); + for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); + // Old servers shut down, while the newer servers remain. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // Ensure old servers shut down properly. + for (int i = 0; i < 3; i++) keepers.get(i).await(); + // Ensure new servers have reconfigured. + for (int i = 3; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Verify written data is preserved. + verifyData(path, configs.get(3)); + + + // Cluster shrinks to a single server. + configs = getConfigs(5, 0, 1, 0); + for (int i = 3; i < 6; i++) keepers.get(i).config = configs.get(i); + for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // We let the remaining server reconfigure the others out before they die. + for (int i = 3; i < 5; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 3; i < 5; i++) keepers.get(i).await(); + verifyData(path, configs.get(5)); + + // Cluster grows to 3 servers again. + configs = getConfigs(5, 0, 3, 2); + for (int i = 5; i < 8; i++) keepers.get(i).config = configs.get(i); + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // Wait for the joiners. + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + verifyData(path, configs.get(7)); + + // Let the remaining servers terminate. + for (int i = 5; i < 8; i++) keepers.get(i).config = null; + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 5; i < 8; i++) keepers.get(i).await(); + } + + static String writeData(ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { + ZooKeeperAdmin admin = createAdmin(config); + List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; + String node = admin.create("/test-node", "hi".getBytes(UTF_8), acl, CreateMode.EPHEMERAL_SEQUENTIAL); + String read = new String(admin.getData(node, false, new Stat()), UTF_8); + assertEquals("hi", read); + return node; + } + + static void verifyData(String path, ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { + for (int i = 0; i < 10; i++) { + try { + assertEquals("hi", new String(createAdmin(config).getData(path, false, new Stat()), UTF_8)); + return; + } + catch (KeeperException.ConnectionLossException e) { + e.printStackTrace(); + Thread.sleep(10 << i); + } + } + } + + static ZooKeeperAdmin createAdmin(ZookeeperServerConfig config) throws IOException { + return new ZooKeeperAdmin(HostName.getLocalhost() + ":" + config.clientPort(), + 10_000, + System.err::println, + new ZkClientConfigBuilder().toConfig()); + } + + static class ZooKeeper { + + final ExecutorService executor = Executors.newSingleThreadExecutor(); + final Phaser phaser = new Phaser(2); + final AtomicReference<Future<?>> future = new AtomicReference<>(); + ZookeeperServerConfig config; + + void run() { + future.set(executor.submit(() -> { + Reconfigurer reconfigurer = new Reconfigurer(new VespaZooKeeperAdminImpl()); + phaser.arriveAndAwaitAdvance(); + while (config != null) { + new ReconfigurableVespaZooKeeperServer(reconfigurer, config); + phaser.arriveAndAwaitAdvance(); // server is now up, let test thread sync here + phaser.arriveAndAwaitAdvance(); // wait before reconfig/teardown to let test thread do stuff + } + reconfigurer.deconstruct(); + })); + } + + void await() throws ExecutionException, InterruptedException, TimeoutException { + future.get().get(30, SECONDS); + } + } + + static List<ZookeeperServerConfig> getConfigs(int removed, int retired, int active, int joining) { + return IntStream.rangeClosed(1, removed + retired + active) + .mapToObj(id -> getConfig(removed, retired, active, joining, id)) + .collect(toList()); + } + + // Config for server #id among retired + active servers, of which the last may be joining, and with offset removed. + static ZookeeperServerConfig getConfig(int removed, int retired, int active, int joining, int id) { + if (id <= removed) + return null; + + Path tempDir = tempDirRoot.resolve("zookeeper-" + id); + return new ZookeeperServerConfig.Builder() + .clientPort(getPorts(id).get(0)) + .dataDir(tempDir.toString()) + .zooKeeperConfigFile(tempDir.resolve("zookeeper.cfg").toString()) + .myid(id) + .myidFile(tempDir.resolve("myid").toString()) + .dynamicReconfiguration(true) + .server(IntStream.rangeClosed(removed + 1, removed + retired + active) + .mapToObj(i -> new ZookeeperServerConfig.Server.Builder() + .id(i) + .clientPort(getPorts(i).get(0)) + .electionPort(getPorts(i).get(1)) + .quorumPort(getPorts(i).get(2)) + .hostname("localhost") + .joining(i - removed > retired + active - joining) + .retired(i - removed <= retired)) + .collect(toList())) + .build(); + } + + static List<Integer> getPorts(int id) { + if (ports.size() < id * 3) { + int previousPort; + if (ports.isEmpty()) { + String[] version = System.getProperty("zk-version").split("\\."); + int versionPortOffset = 0; + for (String part : version) + versionPortOffset = 32 * (versionPortOffset + Integer.parseInt(part)); + previousPort = 20000 + versionPortOffset % 30000; + } + else + previousPort = ports.get(ports.size() - 1); + + for (int i = 0; i < 3; i++) + ports.add(previousPort = nextPort(previousPort)); + } + return ports.subList(id * 3 - 3, id * 3); + } + + static int nextPort(int previousPort) { + for (int j = 1; j <= 30000; j++) { + int port = (previousPort + j); + while (port > 50000) + port -= 30000; + + try (ServerSocket socket = new ServerSocket(port)) { + return socket.getLocalPort(); + } + catch (IOException e) { + System.err.println("Could not bind port " + port + ": " + e); + } + } + throw new RuntimeException("No free ports"); + } + + static Path getTmpDir() { + try { + Path tempDir = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), "vespa-zk-test"); + tempDir.toFile().deleteOnExit(); + return tempDir.toAbsolutePath(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java index f95704f8b58..8b22f658a94 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java @@ -41,6 +41,7 @@ public class Configurator { System.setProperty("zookeeper.authProvider.x509", "com.yahoo.vespa.zookeeper.VespaMtlsAuthenticationProvider"); // Need to set this as a system property, otherwise it will be parsed for _every_ packet and an exception will be thrown (and handled) System.setProperty("zookeeper.globalOutstandingLimit", "1000"); + System.setProperty("zookeeper.snapshot.compression.method", zookeeperServerConfig.snapshotMethod()); } void writeConfigToDisk() { writeConfigToDisk(VespaTlsConfig.fromSystem()); } @@ -85,7 +86,7 @@ public class Configurator { sb.append("reconfigEnabled=true").append("\n"); sb.append("skipACL=yes").append("\n"); ensureThisServerIsRepresented(config.myid(), config.server()); - config.server().forEach(server -> addServerToCfg(sb, server, config.clientPort())); + config.server().forEach(server -> sb.append(serverSpec(server, config.clientPort(), server.joining())).append("\n")); sb.append(new TlsQuorumConfig().createConfig(vespaTlsConfig)); sb.append(new TlsClientServerConfig().createConfig(vespaTlsConfig)); return sb.toString(); @@ -110,7 +111,8 @@ public class Configurator { } } - private void addServerToCfg(StringBuilder sb, ZookeeperServerConfig.Server server, int clientPort) { + static String serverSpec(ZookeeperServerConfig.Server server, int clientPort, boolean joining) { + StringBuilder sb = new StringBuilder(); sb.append("server.") .append(server.id()) .append("=") @@ -119,7 +121,7 @@ public class Configurator { .append(server.quorumPort()) .append(":") .append(server.electionPort()); - if (server.joining()) { + if (joining) { // Servers that are joining an existing cluster must be marked as observers. Note that this will NOT // actually make the server an observer, but prevent it from forming an ensemble independently of the // existing cluster. @@ -129,8 +131,8 @@ public class Configurator { .append("observer"); } sb.append(";") - .append(clientPort) - .append("\n"); + .append(server.clientPort()); + return sb.toString(); } static List<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) { diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java index d4223e4d815..604419c063d 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java @@ -10,14 +10,14 @@ import com.yahoo.yolean.Exceptions; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.function.Consumer; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; + +import static com.yahoo.vespa.zookeeper.Configurator.serverSpec; +import static java.util.stream.Collectors.toList; /** * Starts zookeeper server and supports reconfiguring zookeeper cluster. Keep this as a component @@ -50,17 +50,22 @@ public class Reconfigurer extends AbstractComponent { this.sleeper = Objects.requireNonNull(sleeper); } - void startOrReconfigure(ZookeeperServerConfig newConfig, VespaZooKeeperServer server, - Supplier<QuorumPeer> quorumPeerGetter, Consumer<QuorumPeer> quorumPeerSetter) { + @Override + public void deconstruct() { + shutdown(); + } + + QuorumPeer startOrReconfigure(ZookeeperServerConfig newConfig, VespaZooKeeperServer server, + Supplier<QuorumPeer> quorumPeerCreator) { if (zooKeeperRunner == null) { - peer = quorumPeerGetter.get(); // Obtain the peer from the server. This will be shared with later servers. + peer = quorumPeerCreator.get(); // Obtain the peer from the server. This will be shared with later servers. zooKeeperRunner = startServer(newConfig, server); } - quorumPeerSetter.accept(peer); - if (shouldReconfigure(newConfig)) { + if (newConfig.dynamicReconfiguration()) { reconfigure(newConfig); } + return peer; } ZookeeperServerConfig activeConfig() { @@ -73,42 +78,30 @@ public class Reconfigurer extends AbstractComponent { } } - private boolean shouldReconfigure(ZookeeperServerConfig newConfig) { - if (!newConfig.dynamicReconfiguration()) return false; - if (activeConfig == null) return false; - return !newConfig.equals(activeConfig()); - } - private ZooKeeperRunner startServer(ZookeeperServerConfig zookeeperServerConfig, VespaZooKeeperServer server) { ZooKeeperRunner runner = new ZooKeeperRunner(zookeeperServerConfig, server); activeConfig = zookeeperServerConfig; return runner; } + // TODO jonmv: read dynamic file, discard if old quorum impossible (config file + .dynamic.<id>) + // TODO jonmv: if dynamic file, all unlisted servers are observers; otherwise joiners are observers + // TODO jonmv: wrap Curator in Provider, for Curator shutdown private void reconfigure(ZookeeperServerConfig newConfig) { Instant reconfigTriggered = Instant.now(); - // No point in trying to reconfigure if there is only one server in the new ensemble, - // the others will be shutdown or are about to be shutdown - if (newConfig.server().size() == 1) shutdownAndDie(Duration.ZERO); - - List<String> newServers = difference(servers(newConfig), servers(activeConfig)); - String leavingServerIds = String.join(",", serverIdsDifference(activeConfig, newConfig)); - String joiningServersSpec = String.join(",", newServers); - leavingServerIds = leavingServerIds.isEmpty() ? null : leavingServerIds; - joiningServersSpec = joiningServersSpec.isEmpty() ? null : joiningServersSpec; - log.log(Level.INFO, "Will reconfigure ZooKeeper cluster. \nJoining servers: " + joiningServersSpec + - "\nleaving servers: " + leavingServerIds + + String newServers = String.join(",", servers(newConfig)); + log.log(Level.INFO, "Will reconfigure ZooKeeper cluster." + "\nServers in active config:" + servers(activeConfig) + "\nServers in new config:" + servers(newConfig)); String connectionSpec = localConnectionSpec(activeConfig); Instant now = Instant.now(); - Duration reconfigTimeout = reconfigTimeout(newServers.size()); + Duration reconfigTimeout = reconfigTimeout(newConfig.server().size()); Instant end = now.plus(reconfigTimeout); // Loop reconfiguring since we might need to wait until another reconfiguration is finished before we can succeed for (int attempt = 1; now.isBefore(end); attempt++) { try { Instant reconfigStarted = Instant.now(); - vespaZooKeeperAdmin.reconfigure(connectionSpec, joiningServersSpec, leavingServerIds); + vespaZooKeeperAdmin.reconfigure(connectionSpec, newServers); Instant reconfigEnded = Instant.now(); log.log(Level.INFO, "Reconfiguration completed in " + Duration.between(reconfigTriggered, reconfigEnded) + @@ -147,24 +140,11 @@ public class Reconfigurer extends AbstractComponent { return HostName.getLocalhost() + ":" + config.clientPort(); } - private static List<String> serverIdsDifference(ZookeeperServerConfig oldConfig, ZookeeperServerConfig newConfig) { - return difference(servers(oldConfig), servers(newConfig)).stream() - .map(server -> server.substring(0, server.indexOf('='))) - .collect(Collectors.toList()); - } - private static List<String> servers(ZookeeperServerConfig config) { - // See https://zookeeper.apache.org/doc/r3.6.3/zookeeperReconfig.html#sc_reconfig_clientport for format return config.server().stream() - .map(server -> server.id() + "=" + server.hostname() + ":" + server.quorumPort() + ":" + - server.electionPort() + ";" + config.clientPort()) - .collect(Collectors.toList()); - } - - private static <T> List<T> difference(List<T> list1, List<T> list2) { - List<T> copy = new ArrayList<>(list1); - copy.removeAll(list2); - return copy; + .filter(server -> ! server.retired()) + .map(server -> serverSpec(server, config.clientPort(), false)) + .collect(toList()); } } diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdmin.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdmin.java index 8809dca0def..59c9628bcab 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdmin.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdmin.java @@ -10,7 +10,7 @@ import java.time.Duration; */ public interface VespaZooKeeperAdmin { - void reconfigure(String connectionSpec, String joiningServers, String leavingServers) throws ReconfigException; + void reconfigure(String connectionSpec, String servers) throws ReconfigException; /* Timeout for connecting to ZooKeeper */ default Duration sessionTimeout() { return Duration.ofSeconds(30); } diff --git a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java index 1211624e3d6..760c326cf5d 100644 --- a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java +++ b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java @@ -17,7 +17,6 @@ import java.util.Arrays; import java.util.concurrent.Phaser; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; /** @@ -51,31 +50,26 @@ public class ReconfigurerTest { ZookeeperServerConfig nextConfig = createConfig(5, true); reconfigurer.startOrReconfigure(nextConfig); assertEquals("node1:2181", reconfigurer.connectionSpec()); - assertEquals("3=node3:2182:2183;2181,4=node4:2182:2183;2181", reconfigurer.joiningServers()); - assertNull("No servers are leaving", reconfigurer.leavingServers()); - assertEquals(1, reconfigurer.reconfigurations()); - assertSame(nextConfig, reconfigurer.activeConfig()); - - // No reconfiguration happens with same config - reconfigurer.startOrReconfigure(nextConfig); - assertEquals(1, reconfigurer.reconfigurations()); + assertEquals("server.0=node0:2182:2183;2181,server.1=node1:2182:2183;2181,server.2=node2:2182:2183;2181,server.3=node3:2182:2183;2181,server.4=node4:2182:2183;2181", + reconfigurer.servers()); + assertEquals(2, reconfigurer.reconfigurations()); assertSame(nextConfig, reconfigurer.activeConfig()); // Cluster shrinks nextConfig = createConfig(3, true); reconfigurer.startOrReconfigure(nextConfig); - assertEquals(2, reconfigurer.reconfigurations()); + assertEquals(3, reconfigurer.reconfigurations()); assertEquals("node1:2181", reconfigurer.connectionSpec()); - assertNull("No servers are joining", reconfigurer.joiningServers()); - assertEquals("3,4", reconfigurer.leavingServers()); + assertEquals("server.0=node0:2182:2183;2181,server.1=node1:2182:2183;2181,server.2=node2:2182:2183;2181", + reconfigurer.servers()); assertSame(nextConfig, reconfigurer.activeConfig()); // Cluster loses node1, but node3 joins. Indices are shuffled. nextConfig = createConfig(3, true, 1); reconfigurer.startOrReconfigure(nextConfig); - assertEquals(3, reconfigurer.reconfigurations()); - assertEquals("1=node2:2182:2183;2181,2=node3:2182:2183;2181", reconfigurer.joiningServers()); - assertEquals("1,2", reconfigurer.leavingServers()); + assertEquals(4, reconfigurer.reconfigurations()); + assertEquals("server.0=node0:2182:2183;2181,server.1=node2:2182:2183;2181,server.2=node3:2182:2183;2181", + reconfigurer.servers()); assertSame(nextConfig, reconfigurer.activeConfig()); } @@ -89,9 +83,9 @@ public class ReconfigurerTest { ZookeeperServerConfig nextConfig = createConfig(5, true); reconfigurer.startOrReconfigure(nextConfig); assertEquals("node1:2181", reconfigurer.connectionSpec()); - assertEquals("3=node3:2182:2183;2181,4=node4:2182:2183;2181", reconfigurer.joiningServers()); - assertNull("No servers are leaving", reconfigurer.leavingServers()); - assertEquals(1, reconfigurer.reconfigurations()); + assertEquals("server.0=node0:2182:2183;2181,server.1=node1:2182:2183;2181,server.2=node2:2182:2183;2181,server.3=node3:2182:2183;2181,server.4=node4:2182:2183;2181", + reconfigurer.servers()); + assertEquals(2, reconfigurer.reconfigurations()); assertSame(nextConfig, reconfigurer.activeConfig()); } @@ -112,24 +106,27 @@ public class ReconfigurerTest { reconfigurer.shutdown(); } - private ZookeeperServerConfig createConfig(int numberOfServers, boolean dynamicReconfiguration, int... skipIndices) { - Arrays.sort(skipIndices); + private ZookeeperServerConfig createConfig(int numberOfServers, boolean dynamicReconfiguration, int... retiredIndices) { + Arrays.sort(retiredIndices); ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); builder.myidFile(idFile.getAbsolutePath()); for (int i = 0, index = 0; i < numberOfServers; i++, index++) { - while (Arrays.binarySearch(skipIndices, index) >= 0) index++; - builder.server(newServer(i, "node" + index)); + boolean retired = Arrays.binarySearch(retiredIndices, index) >= 0; + if (retired) i--; + builder.server(newServer(i, "node" + index, retired)); } + builder.myid(0); builder.dynamicReconfiguration(dynamicReconfiguration); return builder.build(); } - private ZookeeperServerConfig.Server.Builder newServer(int id, String hostName) { + private ZookeeperServerConfig.Server.Builder newServer(int id, String hostName, boolean retired) { ZookeeperServerConfig.Server.Builder builder = new ZookeeperServerConfig.Server.Builder(); builder.id(id); builder.hostname(hostName); + builder.retired(retired); return builder; } @@ -142,6 +139,7 @@ public class ReconfigurerTest { private static class TestableReconfigurer extends Reconfigurer implements VespaZooKeeperServer { private final TestableVespaZooKeeperAdmin zooKeeperAdmin; + private final Phaser phaser = new Phaser(2); private QuorumPeer serverPeer; TestableReconfigurer(TestableVespaZooKeeperAdmin zooKeeperAdmin) { @@ -156,19 +154,16 @@ public class ReconfigurerTest { } void startOrReconfigure(ZookeeperServerConfig newConfig) { - startOrReconfigure(newConfig, this, MockQuorumPeer::new, peer -> serverPeer = peer); + serverPeer = startOrReconfigure(newConfig, this, MockQuorumPeer::new); + phaser.arriveAndDeregister(); } String connectionSpec() { return zooKeeperAdmin.connectionSpec; } - String joiningServers() { - return zooKeeperAdmin.joiningServers; - } - - String leavingServers() { - return zooKeeperAdmin.leavingServers; + String servers() { + return zooKeeperAdmin.servers; } int reconfigurations() { @@ -177,10 +172,14 @@ public class ReconfigurerTest { @Override public void shutdown() { + phaser.arriveAndAwaitAdvance(); serverPeer.shutdown(Duration.ofSeconds(1)); } @Override - public void start(Path configFilePath) { serverPeer.start(configFilePath); } + public void start(Path configFilePath) { + phaser.arriveAndAwaitAdvance(); + serverPeer.start(configFilePath); + } @Override public boolean reconfigurable() { @@ -192,8 +191,7 @@ public class ReconfigurerTest { private static class TestableVespaZooKeeperAdmin implements VespaZooKeeperAdmin { String connectionSpec; - String joiningServers; - String leavingServers; + String servers; int reconfigurations = 0; private int failures = 0; @@ -205,12 +203,11 @@ public class ReconfigurerTest { } @Override - public void reconfigure(String connectionSpec, String joiningServers, String leavingServers) throws ReconfigException { + public void reconfigure(String connectionSpec, String servers) throws ReconfigException { if (++attempts < failures) throw new ReconfigException("Reconfig failed"); this.connectionSpec = connectionSpec; - this.joiningServers = joiningServers; - this.leavingServers = leavingServers; + this.servers = servers; this.reconfigurations++; } |