diff options
19 files changed, 203 insertions, 71 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 97d695cead9..508d011d18f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -417,9 +417,29 @@ public class RawRankProfile implements RankProfilesConfig.Producer { properties.add(new Pair<>("vespa.type.query." + queryFeatureType.getKey(), queryFeatureType.getValue())); } if (properties.size() >= 1000000) throw new RuntimeException("Too many rank properties"); + distributeLargeExpressionsAsFiles(properties, largeRankExpressions); return properties; } + private void distributeLargeExpressionsAsFiles(List<Pair<String, String>> properties, LargeRankExpressions largeRankExpressions) { + if (!distributeLargeRankExpressions) return; + for (ListIterator<Pair<String, String>> iter = properties.listIterator(); iter.hasNext();) { + Pair<String, String> property = iter.next(); + String expression = property.getSecond(); + if (expression.length() > largeRankExpressionLimit) { + String propertyName = property.getFirst(); + String functionName = RankingExpression.extractScriptName(propertyName); + if (functionName != null) { + String mangledName = rankprofileName + "." + functionName; + largeRankExpressions.add(new RankExpressionBody(mangledName, ByteBuffer.wrap(expression.getBytes(StandardCharsets.UTF_8)))); + if (useDistributedRankExpressions) { + iter.set(new Pair<>(RankingExpression.propertyExpressionName(functionName), mangledName)); + } + } + } + } + } + private List<Pair<String, String>> deriveRankingPhaseRankProperties(RankingExpression expression, String phase) { List<Pair<String, String>> properties = new ArrayList<>(); if (expression == null) return properties; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java index 396be0adf92..d067b7a5054 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.athenz.api.AthenzGroup; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.api.OktaIdentityToken; @@ -168,6 +169,19 @@ public class ZmsClientMock implements ZmsClient { } @Override + public List<AthenzService> listServices(AthenzDomain athenzDomain) { + return List.of(); + } + + @Override + public void createOrUpdateService(AthenzService athenzService) { + } + + @Override + public void deleteService(AthenzService athenzService) { + } + + @Override public void close() {} private static AthenzDomain getTenantDomain(AthenzResourceName resource) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 134a9b0c6c2..ffc903a309d 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -236,7 +236,7 @@ public class Flags { APPLICATION_ID); public static final UnboundBooleanFlag DRY_RUN_ONNX_ON_SETUP = defineFeatureFlag( - "dry-run-onnx-on-setup", false, + "dry-run-onnx-on-setup", true, List.of("baldersheim"), "2021-06-23", "2021-09-01", "Whether to dry run onnx models on setup for better error checking", "Takes effect on next internal redeployment", diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index 12f4fa23ae9..fe4d581d839 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -373,30 +373,10 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { void gc() { synchronized (writeLock) { - // We remove full days at once and we want to see at least three days to not every only see weekend data - Instant oldestToKeep = clock.instant().minus(Duration.ofDays(4)); - SqlExecutionContext context = newContext(); - int partitions = 0; try { - List<String> removeList = new ArrayList<>(); - for (String dirEntry : dir.list()) { - File partitionDir = new File(dir, dirEntry); - if (!partitionDir.isDirectory()) continue; - - partitions++; - DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneId.of("UTC")); - Instant partitionDay = Instant.from(formatter.parse(dirEntry.substring(0, 10) + "T00:00:00")); - if (partitionDay.isBefore(oldestToKeep)) - removeList.add(dirEntry); - - } - // Remove unless all partitions are old: Removing all partitions "will be supported in the future" - if (removeList.size() < partitions && !removeList.isEmpty()) { - issue("alter table " + name + " drop partition list " + - removeList.stream().map(dir -> "'" + dir + "'").collect(Collectors.joining(",")), - context); - } - } catch (SqlException e) { + issue("alter table " + name + " drop partition where at < dateadd('d', -4, now());", newContext()); + } + catch (SqlException e) { log.log(Level.WARNING, "Failed to gc old metrics data in " + dir + " table " + name, e); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java index 34243f4548f..1c458750569 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDbTest.java @@ -22,6 +22,7 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Tests the Quest metrics db. @@ -163,21 +164,25 @@ public class QuestMetricsDbTest { String dataDir = "data/QuestMetricsDbGc"; IOUtils.recursiveDeleteDir(new File(dataDir)); IOUtils.createDirectory(dataDir + "/metrics"); - ManualClock clock = new ManualClock("2020-10-01T00:00:00"); - QuestMetricsDb db = new QuestMetricsDb(dataDir, clock); + ManualClock clock = new ManualClock(); + int days = 10; // The first metrics are this many days in the past + clock.retreat(Duration.ofDays(10)); Instant startTime = clock.instant(); - int dayOffset = 3; - clock.advance(Duration.ofHours(dayOffset)); - db.addNodeMetrics(nodeTimeseries(24 * 10, Duration.ofHours(1), clock, "host1", "host2", "host3")); - assertEquals(24 * 10, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), - Set.of("host1")).get(0).size()); - db.gc(); - assertEquals(75, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), + QuestMetricsDb db = new QuestMetricsDb(dataDir, clock); + + db.addNodeMetrics(nodeTimeseries(24 * days, Duration.ofHours(1), clock, "host1", "host2", "host3")); + + var application1 = ApplicationId.from("t1", "a1", "i1"); + var cluster1 = new ClusterSpec.Id("cluster1"); + db.addClusterMetrics(application1, Map.of(cluster1, new ClusterMetricSnapshot(clock.instant(), 30.0, 15.0))); + + assertEquals(24 * days, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).size()); + db.gc(); + assertTrue(db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).size() < 24 * 4); db.gc(); // no-op - assertEquals(75, db.getNodeTimeseries(Duration.between(startTime, clock.instant()), - Set.of("host1")).get(0).size()); + assertTrue(db.getNodeTimeseries(Duration.between(startTime, clock.instant()), Set.of("host1")).get(0).size() < 24 * 4); } /** To manually test that we can read existing data */ diff --git a/testutil/src/main/java/com/yahoo/test/ManualClock.java b/testutil/src/main/java/com/yahoo/test/ManualClock.java index ba7d9698d72..296b4a2b360 100644 --- a/testutil/src/main/java/com/yahoo/test/ManualClock.java +++ b/testutil/src/main/java/com/yahoo/test/ManualClock.java @@ -18,7 +18,7 @@ import java.util.concurrent.atomic.AtomicReference; */ public class ManualClock extends Clock { - private AtomicReference<Instant> currentTime = new AtomicReference<>(Instant.now()); + private final AtomicReference<Instant> currentTime = new AtomicReference<>(Instant.now()); @Inject public ManualClock() {} @@ -35,6 +35,11 @@ public class ManualClock extends Clock { currentTime.updateAndGet(time -> time.plus(temporal)); } + /** Move time backwards by the given amount */ + public void retreat(TemporalAmount temporal) { + currentTime.updateAndGet(time -> time.minus(temporal)); + } + public void setInstant(Instant time) { currentTime.set(time); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index 7503b5a39ed..89b72c249bd 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.athenz.api.AthenzGroup; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.api.OktaIdentityToken; @@ -18,13 +19,14 @@ import com.yahoo.vespa.athenz.client.zms.bindings.MembershipEntity; import com.yahoo.vespa.athenz.client.zms.bindings.PolicyEntity; import com.yahoo.vespa.athenz.client.zms.bindings.ProviderResourceGroupRolesRequestEntity; import com.yahoo.vespa.athenz.client.zms.bindings.RoleEntity; +import com.yahoo.vespa.athenz.client.zms.bindings.ServiceEntity; +import com.yahoo.vespa.athenz.client.zms.bindings.ServiceListResponseEntity; import com.yahoo.vespa.athenz.client.zms.bindings.TenancyRequestEntity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.utils.AthenzIdentities; import org.apache.http.Header; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; -import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; import javax.net.ssl.SSLContext; @@ -32,11 +34,9 @@ import java.net.URI; import java.time.Instant; import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; -import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -261,6 +261,34 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { .collect(Collectors.toList()); } + @Override + public List<AthenzService> listServices(AthenzDomain athenzDomain) { + URI uri = zmsUrl.resolve(String.format("domain/%s/service", athenzDomain.getName())); + ServiceListResponseEntity execute = execute(RequestBuilder.get(uri).build(), response -> readEntity(response, ServiceListResponseEntity.class)); + + return execute.services.stream() + .map(serviceName -> new AthenzService(athenzDomain, serviceName)) + .collect(Collectors.toList()); + } + + @Override + public void createOrUpdateService(AthenzService athenzService) { + URI uri = zmsUrl.resolve(String.format("domain/%s/service/%s", athenzService.getDomainName(), athenzService.getName())); + + var serviceEntity = new ServiceEntity(athenzService.getFullName()); + + var request = RequestBuilder.put(uri) + .setEntity(toJsonStringEntity(serviceEntity)) + .build(); + execute(request, response -> readEntity(response, Void.class)); + } + + @Override + public void deleteService(AthenzService athenzService) { + URI uri = zmsUrl.resolve(String.format("domain/%s/service/%s", athenzService.getDomainName(), athenzService.getName())); + execute(RequestBuilder.delete(uri).build(), response -> readEntity(response, Void.class)); + } + private static Header createCookieHeaderWithOktaTokens(OktaIdentityToken identityToken, OktaAccessToken accessToken) { return new BasicHeader("Cookie", String.format("okta_at=%s; okta_it=%s", accessToken.token(), identityToken.token())); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index 03afc9278cc..2807d20f5c6 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -6,6 +6,7 @@ import com.yahoo.vespa.athenz.api.AthenzGroup; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; +import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.AthenzUser; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.api.OktaIdentityToken; @@ -54,5 +55,11 @@ public interface ZmsClient extends AutoCloseable { List<AthenzIdentity> listMembers(AthenzRole athenzRole); + List<AthenzService> listServices(AthenzDomain athenzDomain); + + void createOrUpdateService(AthenzService athenzService); + + void deleteService(AthenzService athenzService); + void close(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ServiceEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ServiceEntity.java new file mode 100644 index 00000000000..aebbc408f69 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ServiceEntity.java @@ -0,0 +1,25 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms.bindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author andreer + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ServiceEntity { + public final String name; + + @JsonCreator + public ServiceEntity(@JsonProperty("name") String name) { + this.name = name; + } + + @JsonGetter("name") + public String name() { + return name; + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ServiceListResponseEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ServiceListResponseEntity.java new file mode 100644 index 00000000000..2d32af8a841 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ServiceListResponseEntity.java @@ -0,0 +1,21 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms.bindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +/** + * @author andreer + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ServiceListResponseEntity { + public final List<String> services; + + @JsonCreator + public ServiceListResponseEntity(@JsonProperty("names") List<String> services) { + this.services = services; + } +} diff --git a/vespa-feed-client/abi-spec.json b/vespa-feed-client/abi-spec.json index e1fddfeefef..808fe152fee 100644 --- a/vespa-feed-client/abi-spec.json +++ b/vespa-feed-client/abi-spec.json @@ -80,7 +80,8 @@ ], "methods": [ "public void success()", - "public void failure()", + "public void failure(ai.vespa.feed.client.HttpResponse)", + "public void failure(java.lang.Throwable)", "public abstract ai.vespa.feed.client.FeedClient$CircuitBreaker$State state()" ], "fields": [ @@ -196,7 +197,8 @@ "methods": [ "public void <init>(java.time.Duration, java.time.Duration)", "public void success()", - "public void failure()", + "public void failure(ai.vespa.feed.client.HttpResponse)", + "public void failure(java.lang.Throwable)", "public ai.vespa.feed.client.FeedClient$CircuitBreaker$State state()" ], "fields": [] diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/ApacheCluster.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/ApacheCluster.java index bf407c60075..6b6d7f82597 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/ApacheCluster.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/ApacheCluster.java @@ -18,12 +18,14 @@ import org.apache.hc.core5.util.Timeout; import javax.net.ssl.SSLContext; import java.io.IOException; import java.net.URI; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hc.core5.http.ssl.TlsCiphers.excludeH2Blacklisted; import static org.apache.hc.core5.http.ssl.TlsCiphers.excludeWeak; @@ -161,6 +163,12 @@ class ApacheCluster implements Cluster { return wrapped.getBodyBytes(); } + @Override + public String toString() { + return "HTTP response with code " + code() + + (body() != null ? " and body '" + new String(body(), UTF_8) + "'" : ""); + } + } } diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java index 65ce8efe107..0089499701f 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/FeedClient.java @@ -68,8 +68,11 @@ public interface FeedClient extends Closeable { /** Called by the client whenever a successful response is obtained. */ default void success() { } - /** Called by the client whenever a transient or fatal error occurs. */ - default void failure() { } + /** Called by the client whenever an error HTTP response is received. */ + default void failure(HttpResponse response) { } + + /** Called by the client whenever an exception occurs trying to obtain a HTTP response. */ + default void failure(Throwable cause) { } /** The current state of the circuit breaker. */ State state(); diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java index 814d0283140..2c4641f38d5 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/GracePeriodCircuitBreaker.java @@ -4,6 +4,7 @@ package ai.vespa.feed.client; import java.time.Duration; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongSupplier; import java.util.logging.Logger; @@ -26,6 +27,7 @@ public class GracePeriodCircuitBreaker implements FeedClient.CircuitBreaker { private final AtomicBoolean halfOpen = new AtomicBoolean(false); private final AtomicBoolean open = new AtomicBoolean(false); private final LongSupplier clock; + private final AtomicReference<String> detail = new AtomicReference<>(); private final long graceMillis; private final long doomMillis; @@ -53,8 +55,18 @@ public class GracePeriodCircuitBreaker implements FeedClient.CircuitBreaker { } @Override - public void failure() { - failingSinceMillis.compareAndSet(NEVER, clock.getAsLong()); + public void failure(HttpResponse response) { + failure(response.toString()); + } + + @Override + public void failure(Throwable cause) { + failure(cause.getMessage()); + } + + private void failure(String detail) { + if (failingSinceMillis.compareAndSet(NEVER, clock.getAsLong())) + this.detail.set(detail); } @Override @@ -63,7 +75,8 @@ public class GracePeriodCircuitBreaker implements FeedClient.CircuitBreaker { if (failingMillis > graceMillis && halfOpen.compareAndSet(false, true)) log.log(INFO, "Circuit breaker is now half-open, as no requests have succeeded for the " + "last " + failingMillis + "ms. The server will be pinged to see if it recovers, " + - "but this client will give up if no successes are observed within " + doomMillis + "ms"); + "but this client will give up if no successes are observed within " + doomMillis + "ms. " + + "First failure was '" + detail.get() + "'."); if (failingMillis > doomMillis && open.compareAndSet(false, true)) log.log(WARNING, "Circuit breaker is now open, after " + doomMillis + "ms of failing request, " + diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java index 667f3489cf7..58112f86090 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/HttpRequestStrategy.java @@ -135,7 +135,7 @@ class HttpRequestStrategy implements RequestStrategy { * or the user has turned off retries for this type of operation. */ private boolean retry(HttpRequest request, Throwable thrown, int attempt) { - breaker.failure(); + breaker.failure(thrown); if ( (thrown instanceof IOException) // General IO problems. || (thrown instanceof CancellationException) // TLS session disconnect. || (thrown instanceof CancelledKeyException)) { // Selection cancelled. @@ -163,7 +163,7 @@ class HttpRequestStrategy implements RequestStrategy { return true; } - breaker.failure(); + breaker.failure(response); logResponse(FINE, response, request, attempt); if (response.code() == 500 || response.code() == 502 || response.code() == 504) { // Hopefully temporary errors. return retry(request, attempt); diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/GracePeriodCircuitBreakerTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/GracePeriodCircuitBreakerTest.java index 9b30ebfd0aa..ebff2689f1a 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/GracePeriodCircuitBreakerTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/GracePeriodCircuitBreakerTest.java @@ -22,6 +22,7 @@ class GracePeriodCircuitBreakerTest { AtomicLong now = new AtomicLong(0); long SECOND = 1000; CircuitBreaker breaker = new GracePeriodCircuitBreaker(now::get, Duration.ofSeconds(1), Duration.ofMinutes(1)); + Throwable error = new Error(); assertEquals(CLOSED, breaker.state(), "Initial state is closed"); @@ -34,7 +35,7 @@ class GracePeriodCircuitBreakerTest { now.addAndGet(100 * SECOND); assertEquals(CLOSED, breaker.state(), "State is closed some time after a success"); - breaker.failure(); + breaker.failure(error); assertEquals(CLOSED, breaker.state(), "State is closed right after a failure"); now.addAndGet(SECOND); @@ -46,7 +47,7 @@ class GracePeriodCircuitBreakerTest { breaker.success(); assertEquals(CLOSED, breaker.state(), "State is closed after a new success"); - breaker.failure(); + breaker.failure(error); now.addAndGet(60 * SECOND); assertEquals(HALF_OPEN, breaker.state(), "State is half-open until doom period has passed"); diff --git a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp index d647a659eb6..414c35864ac 100644 --- a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp +++ b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp @@ -130,7 +130,7 @@ TEST("require that arrays to alloc is capped to min arrays") TEST_DO(assertArraysToAlloc(17, Setup().used(34 * 4).needed(4).minArrays(16))); } -TEST("arrays to alloc considers used elements across all active buffers (no resizing)") +TEST("arrays to alloc considers used elements across all active buffers of same type (no resizing)") { Fixture f(Setup().used(6 * 4)); f.assertArraysToAlloc(6 * 0.5); @@ -140,15 +140,15 @@ TEST("arrays to alloc considers used elements across all active buffers (no resi f.assertArraysToAlloc((6 + 8 + 10) * 0.5); } -TEST("arrays to alloc only considers used elements in current buffer when resizing") +TEST("arrays to alloc considers used elements across all active buffers of same type when resizing") { Fixture f(Setup().used(6 * 4)); f.assertArraysToAlloc(6 * 0.5); f.add_setup(Setup().used(8 * 4).resizing(true)); - f.assertArraysToAlloc(8 + 8 * 0.5); + f.assertArraysToAlloc(8 + (6 + 8) * 0.5); } -TEST("arrays to alloc considers (and subtracts) dead elements across all active buffers (no resizing)") +TEST("arrays to alloc considers (and subtracts) dead elements across all active buffers of same type (no resizing)") { Fixture f(Setup().used(6 * 4).dead(2 * 4)); f.assertArraysToAlloc((6 - 2) * 0.5); @@ -158,12 +158,12 @@ TEST("arrays to alloc considers (and subtracts) dead elements across all active f.assertArraysToAlloc((6 - 2 + 12 - 4 + 20 - 6) * 0.5); } -TEST("arrays to alloc only considers (and subtracts) dead elements in current buffer when resizing") +TEST("arrays to alloc considers (and subtracts) dead elements across all active buffers of same type when resizing") { Fixture f(Setup().used(6 * 4).dead(2 * 4)); f.assertArraysToAlloc((6 - 2) * 0.5); f.add_setup(Setup().used(12 * 4).dead(4 * 4).resizing(true)); - f.assertArraysToAlloc(12 + (12 - 4) * 0.5); + f.assertArraysToAlloc(12 + (6 - 2 + 12 - 4) * 0.5); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp index c4c61250ebe..548ab9199da 100644 --- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp +++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp @@ -563,16 +563,16 @@ TEST(DataStoreTest, require_that_buffer_growth_works) assertGrowStats({ 4, 4, 4, 4, 8, 16, 16, 32, 64, 64 }, { 4 }, 20, 4, 0); // Resize if buffer size is less than 4, min size 0 - assertGrowStats({ 4, 4, 8, 16, 16, 32, 32, 64, 128, 128 }, + assertGrowStats({ 4, 4, 8, 32, 32, 32, 64, 128, 128, 128 }, { 0, 1, 2, 4 }, 4, 0, 4); // Always switch to new buffer, min size 16 assertGrowStats({ 16, 16, 16, 32, 32, 64, 128, 128, 128 }, { 16 }, 68, 16, 0); // Resize if buffer size is less than 16, min size 0 - assertGrowStats({ 16, 32, 32, 64, 64, 128, 128, 128, 128 }, + assertGrowStats({ 16, 32, 32, 128, 128, 128, 128, 128, 128 }, { 0, 1, 2, 4, 8, 16 }, 4, 0, 16); // Resize if buffer size is less than 16, min size 4 - assertGrowStats({ 16, 32, 32, 64, 64, 128, 128, 128, 128 }, + assertGrowStats({ 16, 32, 32, 128, 128, 128, 128, 128, 128 }, { 4, 8, 16 }, 20, 4, 16); // Always switch to new buffer, min size 0 assertGrowStats({ 1, 1, 1, 1, 1, 2, 2, 4, 8, 8, 16, 32 }, @@ -580,7 +580,7 @@ TEST(DataStoreTest, require_that_buffer_growth_works) // Buffers with sizes larger than the huge page size of the mmap allocator. ASSERT_EQ(524288u, HUGE_PAGE_ARRAY_SIZE); - assertGrowStats({ 262144, 524288, 524288, 524288 * 2, 524288 * 2, 524288 * 3, 524288 * 4, 524288 * 5, 524288 * 5, 524288 * 5 }, + assertGrowStats({ 262144, 524288, 524288, 524288 * 3, 524288 * 3, 524288 * 4, 524288 * 5, 524288 * 5, 524288 * 5, 524288 * 5 }, { 0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144 }, 4, 0, HUGE_PAGE_ARRAY_SIZE / 2, HUGE_PAGE_ARRAY_SIZE * 5); } @@ -614,12 +614,12 @@ TEST(DataStoreTest, require_that_offset_in_EntryRefT_is_within_bounds_when_alloc * 4) Cap bytes to alloc to the max offset EntryRef can handle. * The max bytes to alloc is: maxArrays * arraySize * elementSize. */ - assertGrowStats<uint8_t>({8192,16384,16384,32768,32768,65536,98304,98304,98304,98304,98304,98304}, 3); - assertGrowStats<uint8_t>({16384,16384,32768,32768,65536,65536,131072,163840,163840,163840,163840,163840}, 5); - assertGrowStats<uint8_t>({16384,32768,32768,65536,65536,131072,229376,229376,229376,229376,229376,229376}, 7); - assertGrowStats<uint32_t>({8192,16384,16384,32768,32768,65536,98304,98304,98304,98304,98304,98304}, 3); - assertGrowStats<uint32_t>({16384,16384,32768,32768,65536,65536,131072,163840,163840,163840,163840,163840}, 5); - assertGrowStats<uint32_t>({16384,32768,32768,65536,65536,131072,229376,229376,229376,229376,229376,229376}, 7); + assertGrowStats<uint8_t>({8192,16384,16384,65536,65536,98304,98304,98304,98304,98304,98304,98304}, 3); + assertGrowStats<uint8_t>({16384,16384,65536,65536,65536,131072,163840,163840,163840,163840,163840,163840}, 5); + assertGrowStats<uint8_t>({16384,32768,32768,131072,131072,229376,229376,229376,229376,229376,229376,229376}, 7); + assertGrowStats<uint32_t>({8192,16384,16384,65536,65536,98304,98304,98304,98304,98304,98304,98304}, 3); + assertGrowStats<uint32_t>({16384,16384,65536,65536,65536,131072,163840,163840,163840,163840,163840,163840}, 5); + assertGrowStats<uint32_t>({16384,32768,32768,131072,131072,229376,229376,229376,229376,229376,229376,229376}, 7); } namespace { diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp index b04547226c0..eb5865cd68c 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp @@ -108,22 +108,22 @@ size_t BufferTypeBase::calcArraysToAlloc(uint32_t bufferId, ElemCount elemsNeeded, bool resizing) const { size_t reservedElems = getReservedElements(bufferId); + BufferCounts last_bc; BufferCounts bc; if (resizing) { if (!_aggr_counts.empty()) { - bc = _aggr_counts.last_buffer(); + last_bc = _aggr_counts.last_buffer(); } - } else { - bc = _aggr_counts.all_buffers(); } + bc = _aggr_counts.all_buffers(); assert((bc.used_elems % _arraySize) == 0); assert((bc.dead_elems % _arraySize) == 0); assert(bc.used_elems >= bc.dead_elems); - size_t neededArrays = (elemsNeeded + (resizing ? bc.used_elems : reservedElems) + _arraySize - 1) / _arraySize; + size_t neededArrays = (elemsNeeded + (resizing ? last_bc.used_elems : reservedElems) + _arraySize - 1) / _arraySize; size_t liveArrays = (bc.used_elems - bc.dead_elems) / _arraySize; size_t growArrays = (liveArrays * _allocGrowFactor); - size_t usedArrays = bc.used_elems / _arraySize; + size_t usedArrays = last_bc.used_elems / _arraySize; size_t wantedArrays = std::max((resizing ? usedArrays : 0u) + growArrays, static_cast<size_t>(_minArrays)); |