diff options
24 files changed, 269 insertions, 196 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java index 4f2ab2b1734..c06ade1adcf 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ArtifactRepository.java @@ -15,7 +15,7 @@ public interface ArtifactRepository { /** Returns the system application package of the given version. */ byte[] getSystemApplicationPackage(ApplicationId application, ZoneId zone, Version version); - /** Returns the current stable OS version for the given major version */ - StableOsVersion stableOsVersion(int major); + /** Returns the current OS release with the given major version and tag */ + OsRelease osRelease(int major, OsRelease.Tag tag); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/OsRelease.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/OsRelease.java new file mode 100644 index 00000000000..d80b2201810 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/OsRelease.java @@ -0,0 +1,65 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.deployment; + +import com.yahoo.component.Version; + +import java.time.Instant; +import java.util.Objects; + +/** + * An OS release and its tag. + * + * @author mpolden + */ +public class OsRelease { + + private final Version version; + private final Tag tag; + private final Instant taggedAt; + + public OsRelease(Version version, Tag tag, Instant taggedAt) { + this.version = Objects.requireNonNull(version); + this.tag = Objects.requireNonNull(tag); + this.taggedAt = Objects.requireNonNull(taggedAt); + } + + /** The version number */ + public Version version() { + return version; + } + + /** The tag of this */ + public Tag tag() { + return tag; + } + + /** Returns the time this was tagged */ + public Instant taggedAt() { + return taggedAt; + } + + @Override + public String toString() { + return "os release " + version + ", tagged " + tag + " at " + taggedAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OsRelease osRelease = (OsRelease) o; + return version.equals(osRelease.version) && tag == osRelease.tag && taggedAt.equals(osRelease.taggedAt); + } + + @Override + public int hashCode() { + return Objects.hash(version, tag, taggedAt); + } + + /** Known release tags */ + public enum Tag { + latest, + stable, + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/StableOsVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/StableOsVersion.java deleted file mode 100644 index 98bf5d9d0d7..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/StableOsVersion.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.deployment; - -import com.yahoo.component.Version; - -import java.time.Instant; -import java.util.Objects; - -/** - * A stable OS version. - * - * @author mpolden - */ -public class StableOsVersion { - - private final Version version; - private final Instant promotedAt; - - public StableOsVersion(Version version, Instant promotedAt) { - this.version = Objects.requireNonNull(version); - this.promotedAt = Objects.requireNonNull(promotedAt); - } - - /** The version number */ - public Version version() { - return version; - } - - /** Returns the time this was promoted to stable */ - public Instant promotedAt() { - return promotedAt; - } - - @Override - public String toString() { - return "os version " + version + ", promoted at " + promotedAt; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - StableOsVersion that = (StableOsVersion) o; - return version.equals(that.version) && promotedAt.equals(that.promotedAt); - } - - @Override - public int hashCode() { - return Objects.hash(version, promotedAt); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index b2d3e1b780f..aa087f58059 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.StableOsVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.OsRelease; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import java.time.Duration; @@ -67,10 +67,10 @@ public class OsUpgradeScheduler extends ControllerMaintainer { } private Release releaseIn(CloudName cloud) { - boolean useStableRelease = controller().zoneRegistry().zones().reprovisionToUpgradeOs().ofCloud(cloud) + boolean useTaggedRelease = controller().zoneRegistry().zones().reprovisionToUpgradeOs().ofCloud(cloud) .zones().isEmpty(); - if (useStableRelease) { - return new StableRelease(controller().system(), controller().serviceRegistry().artifactRepository()); + if (useTaggedRelease) { + return new TaggedRelease(controller().system(), controller().serviceRegistry().artifactRepository()); } return new CalendarVersionedRelease(controller().system()); } @@ -85,32 +85,37 @@ public class OsUpgradeScheduler extends ControllerMaintainer { } - /** OS release based on a stable tag */ - private static class StableRelease implements Release { + /** OS release based on a tag */ + private static class TaggedRelease implements Release { private final SystemName system; private final ArtifactRepository artifactRepository; - private StableRelease(SystemName system, ArtifactRepository artifactRepository) { + private TaggedRelease(SystemName system, ArtifactRepository artifactRepository) { this.system = Objects.requireNonNull(system); this.artifactRepository = Objects.requireNonNull(artifactRepository); } @Override public Version version(OsVersionTarget currentTarget, Instant now) { - StableOsVersion stableVersion = artifactRepository.stableOsVersion(currentTarget.osVersion().version().getMajor()); - boolean cooldownPassed = stableVersion.promotedAt().isBefore(now.minus(cooldown())); - return cooldownPassed ? stableVersion.version() : currentTarget.osVersion().version(); + OsRelease release = artifactRepository.osRelease(currentTarget.osVersion().version().getMajor(), tag()); + boolean cooldownPassed = !release.taggedAt().plus(cooldown()).isAfter(now); + return cooldownPassed ? release.version() : currentTarget.osVersion().version(); } @Override public Duration upgradeBudget() { - return Duration.ZERO; // Stable releases happen in-place so no budget is required + return Duration.ZERO; // Upgrades to tagged releases happen in-place so no budget is required } - /** The cool-down period that must pass before a stable version can be used */ + /** Returns the release tag tracked by this system */ + private OsRelease.Tag tag() { + return system.isCd() ? OsRelease.Tag.latest : OsRelease.Tag.stable; + } + + /** The cool-down period that must pass before a release can be used */ private Duration cooldown() { - return system.isCd() ? Duration.ZERO : Duration.ofDays(7); + return system.isCd() ? Duration.ofDays(1) : Duration.ZERO; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java index aa4cabb4fe8..dc7010312a2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ArtifactRepositoryMock.java @@ -6,7 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.StableOsVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.OsRelease; import java.util.HashMap; import java.util.Map; @@ -16,7 +16,7 @@ import java.util.Map; */ public class ArtifactRepositoryMock extends AbstractComponent implements ArtifactRepository { - private final Map<Integer, StableOsVersion> stableOsVersions = new HashMap<>(); + private final Map<String, OsRelease> releases = new HashMap<>(); @Override public byte[] getSystemApplicationPackage(ApplicationId application, ZoneId zone, Version version) { @@ -24,14 +24,18 @@ public class ArtifactRepositoryMock extends AbstractComponent implements Artifac } @Override - public StableOsVersion stableOsVersion(int major) { - StableOsVersion version = stableOsVersions.get(major); - if (version == null) throw new IllegalArgumentException("No version set for major " + major); - return version; + public OsRelease osRelease(int major, OsRelease.Tag tag) { + OsRelease release = releases.get(key(major, tag)); + if (release == null) throw new IllegalArgumentException("No version set for major " + major + " with tag " + tag); + return release; } - public void promoteOsVersion(StableOsVersion stableOsVersion) { - stableOsVersions.put(stableOsVersion.version().getMajor(), stableOsVersion); + public void addRelease(OsRelease osRelease) { + releases.put(key(osRelease.version().getMajor(), osRelease.tag()), osRelease); + } + + private static String key(int major, OsRelease.Tag tag) { + return major + "@" + tag.name(); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java index 51bda73025d..300aa86b5ea 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeSchedulerTest.java @@ -3,9 +3,10 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.StableOsVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.OsRelease; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.versions.OsVersionTarget; import org.junit.Test; @@ -68,7 +69,6 @@ public class OsUpgradeSchedulerTest { @Test public void schedule_stable_release() { ControllerTester tester = new ControllerTester(); - OsUpgradeScheduler scheduler = new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)); Instant t0 = Instant.parse("2021-06-21T07:00:00.00Z"); // Inside trigger period tester.clock().setInstant(t0); @@ -77,40 +77,49 @@ public class OsUpgradeSchedulerTest { Version version0 = Version.fromString("8.0"); tester.controller().upgradeOsIn(cloud, version0, Duration.ZERO, false); - // New version is promoted to stable + // Stable release is scheduled immediately Version version1 = Version.fromString("8.1"); - tester.serviceRegistry().artifactRepository().promoteOsVersion(new StableOsVersion(version1, tester.clock().instant())); - scheduler.maintain(); - assertEquals("Target is unchanged as not enough time has passed", version0, - tester.controller().osVersionTarget(cloud).get().osVersion().version()); - - // Enough time passes since promotion of stable release - tester.clock().advance(Duration.ofDays(7).plus(Duration.ofSeconds(1))); - scheduler.maintain(); - OsVersionTarget target0 = tester.controller().osVersionTarget(cloud).get(); - assertEquals(version1, target0.osVersion().version()); - assertEquals("No budget when upgrading to stable release", - Duration.ZERO, target0.upgradeBudget()); - - // Another version is promoted, but target remains unchanged as the release hasn't aged enough - tester.clock().advance(Duration.ofDays(1)); - Version version2 = Version.fromString("8.2"); - tester.serviceRegistry().artifactRepository().promoteOsVersion(new StableOsVersion(version2, tester.clock().instant())); - scheduler.maintain(); - OsVersionTarget target1 = tester.controller().osVersionTarget(cloud).get(); - assertEquals("Target is unchanged as not enough time has passed", version1, - target1.osVersion().version()); - assertEquals("Target is not re-scheduled", target0.scheduledAt(), target1.scheduledAt()); + tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.stable, + tester.clock().instant())); + scheduleUpgradeAfter(Duration.ZERO, version1, tester); // A newer version is triggered manually Version version3 = Version.fromString("8.3"); tester.controller().upgradeOsIn(cloud, version3, Duration.ZERO, false); - // Enough time passes for stable version to be promoted. Nothing happens as stable is now before the manually - // triggered version - tester.clock().advance(Duration.ofDays(7).plus(Duration.ofSeconds(1))); - scheduler.maintain(); - assertEquals(version3, tester.controller().osVersionTarget(cloud).get().osVersion().version()); + // Nothing happens in next iteration as tagged release is older than manually triggered version + scheduleUpgradeAfter(Duration.ofDays(7), version3, tester); + } + + @Test + public void schedule_latest_release_in_cd() { + ControllerTester tester = new ControllerTester(SystemName.cd); + Instant t0 = Instant.parse("2021-06-21T07:00:00.00Z"); // Inside trigger period + tester.clock().setInstant(t0); + + // Set initial target + CloudName cloud = tester.controller().clouds().iterator().next(); + Version version0 = Version.fromString("8.0"); + tester.controller().upgradeOsIn(cloud, version0, Duration.ZERO, false); + + // Latest release is not scheduled immediately + Version version1 = Version.fromString("8.1"); + tester.serviceRegistry().artifactRepository().addRelease(new OsRelease(version1, OsRelease.Tag.latest, + tester.clock().instant())); + scheduleUpgradeAfter(Duration.ZERO, version0, tester); + + // Cooldown period passes and latest release is scheduled + scheduleUpgradeAfter(Duration.ofDays(1), version1, tester); + } + + private void scheduleUpgradeAfter(Duration duration, Version version, ControllerTester tester) { + tester.clock().advance(duration); + new OsUpgradeScheduler(tester.controller(), Duration.ofDays(1)).maintain(); + CloudName cloud = tester.controller().clouds().iterator().next(); + OsVersionTarget target = tester.controller().osVersionTarget(cloud).get(); + assertEquals(version, target.osVersion().version()); + assertEquals("No budget when scheduling a tagged release", + Duration.ZERO, target.upgradeBudget()); } private static ZoneApi zone(String id, CloudName cloud) { diff --git a/dist/STLExtras.h.diff b/dist/STLExtras.h.diff new file mode 100644 index 00000000000..40f6a2a12ba --- /dev/null +++ b/dist/STLExtras.h.diff @@ -0,0 +1,20 @@ +--- STLExtras.h.orig 2021-01-28 01:34:01.000000000 +0100 ++++ STLExtras.h 2021-03-03 22:18:46.028992086 +0100 +@@ -1820,7 +1820,7 @@ + result_pair(std::size_t Index, IterOfRange<R> Iter) + : Index(Index), Iter(Iter) {} + +- result_pair<R>(const result_pair<R> &Other) ++ result_pair(const result_pair<R> &Other) + : Index(Other.Index), Iter(Other.Iter) {} + result_pair<R> &operator=(const result_pair<R> &Other) { + Index = Other.Index; +@@ -1870,7 +1870,7 @@ + return Result.Iter == RHS.Result.Iter; + } + +- enumerator_iter<R>(const enumerator_iter<R> &Other) : Result(Other.Result) {} ++ enumerator_iter(const enumerator_iter<R> &Other) : Result(Other.Result) {} + enumerator_iter<R> &operator=(const enumerator_iter<R> &Other) { + Result = Other.Result; + return *this; diff --git a/dist/vespa.spec b/dist/vespa.spec index 6c3f68f6f74..724cffcc7f1 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -549,6 +549,12 @@ nearest neighbor search used for low-level benchmarking. %setup -c -D -T %else %setup -q +%if ( 0%{?el8} || 0%{?fc34} ) && %{_vespa_llvm_version} < 13 +if grep -qs 'result_pair<R>(' /usr/include/llvm/ADT/STLExtras.h +then + patch /usr/include/llvm/ADT/STLExtras.h < dist/STLExtras.h.diff +fi +%endif echo '%{version}' > VERSION case '%{version}' in *.0) diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp index 072b76c6443..ce85517ee09 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp +++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp @@ -121,9 +121,16 @@ TEST_F(DiskMemUsageFilterTest, both_disk_limit_and_memory_limit_can_be_reached) "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}"); } +TEST_F(DiskMemUsageFilterTest, transient_disk_usage_is_tracked_in_usage_state_and_metrics) +{ + _filter.set_transient_resource_usage({40, 0}); + EXPECT_EQ(0.4, _filter.usageState().transient_disk_usage()); + EXPECT_EQ(0.4, _filter.get_metrics().get_transient_disk_usage()); +} + TEST_F(DiskMemUsageFilterTest, transient_memory_usage_is_tracked_in_usage_state_and_metrics) { - _filter.set_transient_resource_usage(200, 0); + _filter.set_transient_resource_usage({0, 200}); EXPECT_EQ(0.2, _filter.usageState().transient_memory_usage()); EXPECT_EQ(0.2, _filter.get_metrics().get_transient_memory_usage()); } diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp index 7fa51bb8b95..0c80553e1e7 100644 --- a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp +++ b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp @@ -34,8 +34,7 @@ public: : _memory_usage(memory_usage), _disk_usage(disk_usage) {} - size_t get_transient_memory_usage() const override { return _memory_usage; } - size_t get_transient_disk_usage() const override { return _disk_usage; } + TransientResourceUsage get_transient_resource_usage() const override { return {_disk_usage, _memory_usage}; } }; struct DiskMemUsageSamplerTest : public ::testing::Test { @@ -46,7 +45,7 @@ struct DiskMemUsageSamplerTest : public ::testing::Test { 50ms, make_hw_info())) { sampler.add_transient_usage_provider(std::make_shared<MyProvider>(50, 200)); - sampler.add_transient_usage_provider(std::make_shared<MyProvider>(100, 199)); + sampler.add_transient_usage_provider(std::make_shared<MyProvider>(100, 150)); } const DiskMemUsageFilter& filter() const { return sampler.writeFilter(); } }; @@ -56,7 +55,7 @@ TEST_F(DiskMemUsageSamplerTest, resource_usage_is_sampled) // Poll for up to 20 seconds to get a sample. size_t i = 0; for (; i < static_cast<size_t>(20s / 50ms); ++i) { - if (filter().get_transient_memory_usage() > 0) { + if (filter().get_transient_resource_usage().memory() > 0) { break; } std::this_thread::sleep_for(50ms); @@ -70,10 +69,10 @@ TEST_F(DiskMemUsageSamplerTest, resource_usage_is_sampled) EXPECT_EQ(filter().getMemoryStats().getAnonymousRss(), 0); #endif EXPECT_GT(filter().getDiskUsedSize(), 0); - EXPECT_EQ(150, filter().get_transient_memory_usage()); + EXPECT_EQ(150, filter().get_transient_resource_usage().memory()); EXPECT_EQ(150.0 / memory_size_bytes, filter().usageState().transient_memory_usage()); - EXPECT_EQ(200, filter().get_transient_disk_usage()); - EXPECT_EQ(200.0 / disk_size_bytes, filter().get_relative_transient_disk_usage()); + EXPECT_EQ(350, filter().get_transient_resource_usage().disk()); + EXPECT_EQ(350.0 / disk_size_bytes, filter().usageState().transient_disk_usage()); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h index 5aeb84feb2c..833b6519046 100644 --- a/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h +++ b/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h @@ -7,6 +7,32 @@ namespace proton { /** + * Class containing transient disk and memory usage (in bytes). + */ +class TransientResourceUsage { +private: + size_t _disk; + size_t _memory; + +public: + TransientResourceUsage() noexcept + : _disk(0), + _memory(0) + {} + TransientResourceUsage(size_t disk_in, + size_t memory_in) noexcept + : _disk(disk_in), + _memory(memory_in) + {} + size_t disk() const noexcept { return _disk; } + size_t memory() const noexcept { return _memory; } + void merge(const TransientResourceUsage& rhs) { + _disk += rhs.disk(); + _memory += rhs.memory(); + } +}; + +/** * Interface class providing a snapshot of transient resource usage. * * E.g. the memory used by the memory index and extra disk needed for running disk index fusion. @@ -15,8 +41,7 @@ namespace proton { class ITransientResourceUsageProvider { public: virtual ~ITransientResourceUsageProvider() = default; - virtual size_t get_transient_memory_usage() const = 0; - virtual size_t get_transient_disk_usage() const = 0; + virtual TransientResourceUsage get_transient_resource_usage() const = 0; }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp index 27eb636b475..8928045b814 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp @@ -131,6 +131,7 @@ DiskMemUsageFilter::recalcState(const Guard &guard) } DiskMemUsageState dmstate(ResourceUsageState(_config._diskLimit, diskUsed), ResourceUsageState(_config._memoryLimit, memoryUsed), + get_relative_transient_disk_usage(guard), get_relative_transient_memory_usage(guard)); notifyDiskMemUsage(guard, dmstate); } @@ -153,9 +154,14 @@ DiskMemUsageFilter::getDiskUsedRatio(const Guard&) const double DiskMemUsageFilter::get_relative_transient_memory_usage(const Guard&) const { - return static_cast<double>(_transient_memory_usage) / _hwInfo.memory().sizeBytes(); + return static_cast<double>(_transient_usage.memory()) / _hwInfo.memory().sizeBytes(); } +double +DiskMemUsageFilter::get_relative_transient_disk_usage(const Guard&) const +{ + return static_cast<double>(_transient_usage.disk()) / _hwInfo.disk().sizeBytes(); +} DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo) : _lock(), @@ -163,7 +169,7 @@ DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo) _acceptWrite(true), _memoryStats(), _diskUsedSizeBytes(), - _transient_memory_usage(0u), + _transient_usage(), _config(), _state(), _dmstate(), @@ -190,11 +196,10 @@ DiskMemUsageFilter::setDiskUsedSize(uint64_t diskUsedSizeBytes) } void -DiskMemUsageFilter::set_transient_resource_usage(size_t transient_memory_usage, size_t transient_disk_usage) +DiskMemUsageFilter::set_transient_resource_usage(const TransientResourceUsage& transient_usage) { Guard guard(_lock); - _transient_memory_usage = transient_memory_usage; - _transient_disk_usage = transient_disk_usage; + _transient_usage = transient_usage; recalcState(guard); } @@ -220,25 +225,11 @@ DiskMemUsageFilter::getDiskUsedSize() const return _diskUsedSizeBytes; } -size_t -DiskMemUsageFilter::get_transient_memory_usage() const -{ - Guard guard(_lock); - return _transient_memory_usage; -} - -size_t -DiskMemUsageFilter::get_transient_disk_usage() const -{ - Guard guard(_lock); - return _transient_disk_usage; -} - -double -DiskMemUsageFilter::get_relative_transient_disk_usage() const +TransientResourceUsage +DiskMemUsageFilter::get_transient_resource_usage() const { Guard guard(_lock); - return static_cast<double>(_transient_disk_usage) / _hwInfo.disk().sizeBytes(); + return _transient_usage; } DiskMemUsageFilter::Config diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h index fb18a12084f..ac328c3ddea 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h @@ -6,6 +6,7 @@ #include "disk_mem_usage_state.h" #include "disk_mem_usage_metrics.h" #include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/searchcore/proton/common/i_transient_resource_usage_provider.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> #include <vespa/vespalib/util/process_memory_stats.h> #include <atomic> @@ -47,8 +48,7 @@ private: // Following member variables are protected by _lock vespalib::ProcessMemoryStats _memoryStats; uint64_t _diskUsedSizeBytes; - size_t _transient_memory_usage; - size_t _transient_disk_usage; + TransientResourceUsage _transient_usage; Config _config; State _state; DiskMemUsageState _dmstate; @@ -59,6 +59,7 @@ private: double getMemoryUsedRatio(const Guard &guard) const; double getDiskUsedRatio(const Guard &guard) const; double get_relative_transient_memory_usage(const Guard& guard) const; + double get_relative_transient_disk_usage(const Guard& guard) const; void notifyDiskMemUsage(const Guard &guard, DiskMemUsageState state); public: @@ -66,13 +67,11 @@ public: ~DiskMemUsageFilter() override; void setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in); void setDiskUsedSize(uint64_t diskUsedSizeBytes); - void set_transient_resource_usage(size_t transient_memory_usage, size_t transient_disk_usage); + void set_transient_resource_usage(const TransientResourceUsage& transient_usage); void setConfig(Config config); vespalib::ProcessMemoryStats getMemoryStats() const; uint64_t getDiskUsedSize() const; - size_t get_transient_memory_usage() const; - size_t get_transient_disk_usage() const; - double get_relative_transient_disk_usage() const; + TransientResourceUsage get_transient_resource_usage() const; Config getConfig() const; const HwInfo &getHwInfo() const { return _hwInfo; } DiskMemUsageState usageState() const; diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp index 562e20a864c..230593c2c1d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp @@ -14,6 +14,7 @@ DiskMemUsageMetrics::DiskMemUsageMetrics() noexcept DiskMemUsageMetrics::DiskMemUsageMetrics(const DiskMemUsageState &usage_state) noexcept : _disk_usage(usage_state.diskState().usage()), _disk_utilization(usage_state.diskState().utilization()), + _transient_disk_usage(usage_state.transient_disk_usage()), _memory_usage(usage_state.memoryState().usage()), _memory_utilization(usage_state.memoryState().utilization()), _transient_memory_usage(usage_state.transient_memory_usage()) @@ -25,6 +26,7 @@ DiskMemUsageMetrics::merge(const DiskMemUsageState &usage_state) noexcept { _disk_usage = std::max(_disk_usage, usage_state.diskState().usage()); _disk_utilization = std::max(_disk_utilization, usage_state.diskState().utilization()); + _transient_disk_usage = std::max(_transient_disk_usage, usage_state.transient_disk_usage()); _memory_usage = std::max(_memory_usage, usage_state.memoryState().usage()); _memory_utilization = std::max(_memory_utilization, usage_state.memoryState().utilization()); _transient_memory_usage = std::max(_transient_memory_usage, usage_state.transient_memory_usage()); diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h index cb084b90a11..cb97eb4c891 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h @@ -14,6 +14,7 @@ class DiskMemUsageMetrics { double _disk_usage; double _disk_utilization; + double _transient_disk_usage; double _memory_usage; double _memory_utilization; double _transient_memory_usage; @@ -24,6 +25,7 @@ public: void merge(const DiskMemUsageState &usage_state) noexcept; double get_disk_usage() const noexcept { return _disk_usage; } double get_disk_utilization() const noexcept { return _disk_utilization; } + double get_transient_disk_usage() const noexcept { return _transient_disk_usage; } double get_memory_usage() const noexcept { return _memory_usage; } double get_memory_utilization() const noexcept { return _memory_utilization; } double get_transient_memory_usage() const noexcept { return _transient_memory_usage; } diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp index cfed4c0522a..2dc749ce26b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp @@ -125,16 +125,14 @@ DiskMemUsageSampler::sampleMemoryUsage() void DiskMemUsageSampler::sample_transient_resource_usage() { - size_t transient_memory_usage_sum = 0; - size_t max_transient_disk_usage = 0; + TransientResourceUsage transient_usage; { std::lock_guard<std::mutex> guard(_lock); for (auto provider : _transient_usage_providers) { - transient_memory_usage_sum += provider->get_transient_memory_usage(); - max_transient_disk_usage = std::max(max_transient_disk_usage, provider->get_transient_disk_usage()); + transient_usage.merge(provider->get_transient_resource_usage()); } } - _filter.set_transient_resource_usage(transient_memory_usage_sum, max_transient_disk_usage); + _filter.set_transient_resource_usage(transient_usage); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h index 769086574f0..b205b441bcf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h @@ -8,27 +8,31 @@ namespace proton { /** * Class used to describe state of disk and memory usage relative to configured limits. - * In addition relative transient memory usage is tracked. + * In addition relative transient disk and memory usage are tracked. */ class DiskMemUsageState { ResourceUsageState _diskState; ResourceUsageState _memoryState; + double _transient_disk_usage; double _transient_memory_usage; public: DiskMemUsageState() = default; DiskMemUsageState(const ResourceUsageState &diskState_, const ResourceUsageState &memoryState_, + double transient_disk_usage_ = 0, double transient_memory_usage_ = 0) : _diskState(diskState_), _memoryState(memoryState_), + _transient_disk_usage(transient_disk_usage_), _transient_memory_usage(transient_memory_usage_) { } bool operator==(const DiskMemUsageState &rhs) const { return ((_diskState == rhs._diskState) && (_memoryState == rhs._memoryState) && + (_transient_disk_usage == rhs._transient_disk_usage) && (_transient_memory_usage == rhs._transient_memory_usage)); } bool operator!=(const DiskMemUsageState &rhs) const { @@ -36,6 +40,7 @@ public: } const ResourceUsageState &diskState() const { return _diskState; } const ResourceUsageState &memoryState() const { return _memoryState; } + double transient_disk_usage() const { return _transient_disk_usage; } double transient_memory_usage() const { return _transient_memory_usage; } bool aboveDiskLimit(double resourceLimitFactor) const { return diskState().aboveLimit(resourceLimitFactor); } bool aboveMemoryLimit(double resourceLimitFactor) const { return memoryState().aboveLimit(resourceLimitFactor); } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index b9903d626aa..f052d663ba6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -97,14 +97,11 @@ public: DocumentDBResourceUsageProvider(const DocumentDB& doc_db) noexcept : _doc_db(doc_db) {} - size_t get_transient_memory_usage() const override { - return _doc_db.getReadySubDB()->getSearchableStats().memoryUsage().allocatedBytes(); - } - size_t get_transient_disk_usage() const override { - // We estimate the transient disk usage for the next disk index fusion - // as the size of the largest disk index. - // TODO: Change this to actually measure the size of the fusion disk index(es). - return _doc_db.getReadySubDB()->getSearchableStats().max_component_size_on_disk(); + TransientResourceUsage get_transient_resource_usage() const override { + // Transient disk usage is measured as the total disk usage of all current fusion indexes. + // Transient memory usage is measured as the total memory usage of all memory indexes. + auto stats = _doc_db.getReadySubDB()->getSearchableStats(); + return {stats.fusion_size_on_disk(), stats.memoryUsage().allocatedBytes()}; } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 7c52d3d96ff..b128fe16e5e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -758,7 +758,7 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) metrics.resourceUsage.memory.set(dm_metrics.get_memory_usage()); metrics.resourceUsage.memoryUtilization.set(dm_metrics.get_memory_utilization()); metrics.resourceUsage.transient_memory.set(dm_metrics.get_transient_memory_usage()); - metrics.resourceUsage.transient_disk.set(usageFilter.get_relative_transient_disk_usage()); + metrics.resourceUsage.transient_disk.set(dm_metrics.get_transient_disk_usage()); metrics.resourceUsage.memoryMappings.set(usageFilter.getMemoryStats().getMappingsCount()); metrics.resourceUsage.openFileDescriptors.set(FastOS_File::count_open_files()); metrics.resourceUsage.feedingBlocked.set((usageFilter.acceptWriteOperation() ? 0.0 : 1.0)); diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp index 4b5e97a827b..3750c5afb5f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp @@ -44,16 +44,16 @@ ResourceUsageExplorer::get_state(const vespalib::slime::Inserter &inserter, bool disk.setDouble("usage", usageState.diskState().usage()); disk.setDouble("limit", usageState.diskState().limit()); disk.setDouble("utilization", usageState.diskState().utilization()); + disk.setDouble("transient", usageState.transient_disk_usage()); convertDiskStatsToSlime(_usage_filter.getHwInfo(), _usage_filter.getDiskUsedSize(), disk.setObject("stats")); Cursor &memory = object.setObject("memory"); memory.setDouble("usage", usageState.memoryState().usage()); memory.setDouble("limit", usageState.memoryState().limit()); memory.setDouble("utilization", usageState.memoryState().utilization()); + memory.setDouble("transient", usageState.transient_memory_usage()); memory.setLong("physicalMemory", _usage_filter.getHwInfo().memory().sizeBytes()); convertMemoryStatsToSlime(_usage_filter.getMemoryStats(), memory.setObject("stats")); - size_t transient_memory = _usage_filter.get_transient_memory_usage(); - memory.setLong("transient", transient_memory); Cursor &address_space = object.setObject("attribute_address_space"); address_space.setDouble("usage", attr_usage.get_usage()); diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index a2bd19c3d29..6489cb156ce 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -117,6 +117,7 @@ public: { return _index->createBlueprint(requestContext, fields, term); } + // TODO: Calculate the total disk size of current fusion indexes and set fusion_size_on_disk(). search::SearchableStats getSearchableStats() const override { return _index->getSearchableStats(); } search::SerialNum getSerialNum() const override { return _index->getSerialNum(); diff --git a/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp b/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp index b89324c71f5..ed857d5776b 100644 --- a/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp +++ b/searchlib/src/tests/util/searchable_stats/searchable_stats_test.cpp @@ -7,43 +7,35 @@ LOG_SETUP("searchable_stats_test"); using namespace search; -TEST(SearchableStatsTest, merge_also_tracks_max_size_on_disk_for_component) +TEST(SearchableStatsTest, stats_can_be_merged) { SearchableStats stats; EXPECT_EQ(0u, stats.memoryUsage().allocatedBytes()); EXPECT_EQ(0u, stats.docsInMemory()); EXPECT_EQ(0u, stats.sizeOnDisk()); - EXPECT_EQ(0u, stats.max_component_size_on_disk()); + EXPECT_EQ(0u, stats.fusion_size_on_disk()); { SearchableStats rhs; EXPECT_EQ(&rhs.memoryUsage(vespalib::MemoryUsage(100,0,0,0)), &rhs); EXPECT_EQ(&rhs.docsInMemory(10), &rhs); EXPECT_EQ(&rhs.sizeOnDisk(1000), &rhs); - EXPECT_EQ(1000u, rhs.max_component_size_on_disk()); + EXPECT_EQ(&rhs.fusion_size_on_disk(500), &rhs); EXPECT_EQ(&stats.merge(rhs), &stats); } EXPECT_EQ(100u, stats.memoryUsage().allocatedBytes()); EXPECT_EQ(10u, stats.docsInMemory()); EXPECT_EQ(1000u, stats.sizeOnDisk()); - EXPECT_EQ(1000u, stats.max_component_size_on_disk()); + EXPECT_EQ(500u, stats.fusion_size_on_disk()); stats.merge(SearchableStats() .memoryUsage(vespalib::MemoryUsage(150,0,0,0)) .docsInMemory(15) - .sizeOnDisk(1500)); + .sizeOnDisk(1500) + .fusion_size_on_disk(800)); EXPECT_EQ(250u, stats.memoryUsage().allocatedBytes()); EXPECT_EQ(25u, stats.docsInMemory()); EXPECT_EQ(2500u, stats.sizeOnDisk()); - EXPECT_EQ(1500u, stats.max_component_size_on_disk()); - - stats.merge(SearchableStats() - .memoryUsage(vespalib::MemoryUsage(120,0,0,0)) - .docsInMemory(12) - .sizeOnDisk(1200)); - EXPECT_EQ(370u, stats.memoryUsage().allocatedBytes()); - EXPECT_EQ(37u, stats.docsInMemory()); - EXPECT_EQ(3700u, stats.sizeOnDisk()); - EXPECT_EQ(1500u, stats.max_component_size_on_disk()); + EXPECT_EQ(1300u, stats.fusion_size_on_disk()); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchlib/util/searchable_stats.h b/searchlib/src/vespa/searchlib/util/searchable_stats.h index 970076c12d0..e785a4c4483 100644 --- a/searchlib/src/vespa/searchlib/util/searchable_stats.h +++ b/searchlib/src/vespa/searchlib/util/searchable_stats.h @@ -15,11 +15,11 @@ class SearchableStats private: vespalib::MemoryUsage _memoryUsage; size_t _docsInMemory; - size_t _sizeOnDisk; - size_t _max_component_size_on_disk; + size_t _sizeOnDisk; // in bytes + size_t _fusion_size_on_disk; // in bytes public: - SearchableStats() : _memoryUsage(), _docsInMemory(0), _sizeOnDisk(0), _max_component_size_on_disk(0) {} + SearchableStats() : _memoryUsage(), _docsInMemory(0), _sizeOnDisk(0), _fusion_size_on_disk(0) {} SearchableStats &memoryUsage(const vespalib::MemoryUsage &usage) { _memoryUsage = usage; return *this; @@ -32,22 +32,20 @@ public: size_t docsInMemory() const { return _docsInMemory; } SearchableStats &sizeOnDisk(size_t value) { _sizeOnDisk = value; - _max_component_size_on_disk = value; return *this; } size_t sizeOnDisk() const { return _sizeOnDisk; } - - /** - * Returns the max disk size used by a single Searchable component, - * e.g. among the components that are merged into a SearchableStats instance via merge(). - */ - size_t max_component_size_on_disk() const { return _max_component_size_on_disk; } + SearchableStats& fusion_size_on_disk(size_t value) { + _fusion_size_on_disk = value; + return *this; + } + size_t fusion_size_on_disk() const { return _fusion_size_on_disk; } SearchableStats &merge(const SearchableStats &rhs) { _memoryUsage.merge(rhs._memoryUsage); _docsInMemory += rhs._docsInMemory; _sizeOnDisk += rhs._sizeOnDisk; - _max_component_size_on_disk = std::max(_max_component_size_on_disk, rhs._sizeOnDisk); + _fusion_size_on_disk += rhs._fusion_size_on_disk; return *this; } }; diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java index 63f4ade952a..40bfb70109d 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/mock/MockCuratorFramework.java @@ -1239,25 +1239,25 @@ public class MockCuratorFramework implements CuratorFramework { } @Override - public TransactionCreateBuilder create() { + public TransactionCreateBuilder<CuratorTransactionBridge> create() { ensureNotCommitted(); return new MockTransactionCreateBuilder(); } @Override - public TransactionDeleteBuilder delete() { + public TransactionDeleteBuilder<CuratorTransactionBridge> delete() { ensureNotCommitted(); return new MockTransactionDeleteBuilder(); } @Override - public TransactionSetDataBuilder setData() { + public TransactionSetDataBuilder<CuratorTransactionBridge> setData() { ensureNotCommitted(); return new MockTransactionSetDataBuilder(); } @Override - public TransactionCheckBuilder check() { + public TransactionCheckBuilder<CuratorTransactionBridge> check() { ensureNotCommitted(); throw new UnsupportedOperationException("Not implemented in MockCurator"); } @@ -1266,7 +1266,7 @@ public class MockCuratorFramework implements CuratorFramework { if (committed) throw new IllegalStateException("transaction already committed"); } - private class MockTransactionCreateBuilder implements TransactionCreateBuilder { + private class MockTransactionCreateBuilder implements TransactionCreateBuilder<CuratorTransactionBridge> { private CreateMode createMode = CreateMode.PERSISTENT; @@ -1294,22 +1294,22 @@ public class MockCuratorFramework implements CuratorFramework { } @Override - public TransactionCreateBuilder2 withTtl(long l) { + public TransactionCreateBuilder2<CuratorTransactionBridge> withTtl(long l) { return this; } @Override - public Object withACL(List list, boolean b) { + public MockTransactionCreateBuilder withACL(List<ACL> list, boolean b) { return this; } @Override - public Object withACL(List list) { + public MockTransactionCreateBuilder withACL(List<ACL> list) { return this; } } - private class MockTransactionDeleteBuilder implements TransactionDeleteBuilder { + private class MockTransactionDeleteBuilder implements TransactionDeleteBuilder<CuratorTransactionBridge> { @Override public Pathable<CuratorTransactionBridge> withVersion(int i) { @@ -1324,7 +1324,7 @@ public class MockCuratorFramework implements CuratorFramework { } - private class MockTransactionSetDataBuilder implements TransactionSetDataBuilder { + private class MockTransactionSetDataBuilder implements TransactionSetDataBuilder<CuratorTransactionBridge> { @Override public VersionPathAndBytesable<CuratorTransactionBridge> compressed() { |