diff options
14 files changed, 80 insertions, 63 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index abcf98c38a0..9c68fb95cce 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -48,6 +48,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.Reader; import java.io.StringReader; +import java.nio.file.Files; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; @@ -61,6 +62,7 @@ import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.text.Lowercase.toLowerCase; @@ -595,6 +597,12 @@ public class FilesApplicationPackage implements ApplicationPackage { transformer.transform(new DOMSource(document), new StreamResult(outputStream)); } } catch (TransformerException |ParserConfigurationException | SAXException e) { + // TODO: 2020-11-09, debugging what seems like missing or empty file while preprocessing. Remove afterwards + if (inputXml.canRead()) { + log.log(Level.WARNING, "Error preprocessing " + inputXml.getAbsolutePath() + ", file content: " + IOUtils.readFile(inputXml)); + java.nio.file.Path dir = inputXml.getParentFile().toPath(); + log.log(Level.INFO, "Files in " + dir + ":" + Files.list(dir).map(java.nio.file.Path::toString).collect(Collectors.joining(","))); + } throw new RuntimeException("Error preprocessing " + inputXml.getAbsolutePath() + ": " + e.getMessage(), e); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java index fae52f318bd..ed28bec0d81 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationReindexing.java @@ -42,14 +42,14 @@ public class ApplicationReindexing implements Reindexing { /** Returns a copy of this with common reindexing for the given cluster ready at the given instant. */ public ApplicationReindexing withReady(String cluster, Instant readyAt) { - Cluster current = clusters.getOrDefault(cluster, Cluster.empty); + Cluster current = clusters.getOrDefault(cluster, Cluster.ready(common)); Cluster modified = new Cluster(new Status(readyAt), current.pending, current.ready); return new ApplicationReindexing(common, with(cluster, modified, clusters)); } /** Returns a copy of this with reindexing for the given document type in the given cluster ready at the given instant. */ public ApplicationReindexing withReady(String cluster, String documentType, Instant readyAt) { - Cluster current = clusters.getOrDefault(cluster, Cluster.empty); + Cluster current = clusters.getOrDefault(cluster, Cluster.ready(common)); Cluster modified = new Cluster(current.common, without(documentType, current.pending), with(documentType, new Status(readyAt), current.ready)); @@ -58,7 +58,7 @@ public class ApplicationReindexing implements Reindexing { /** Returns a copy of this with a pending reindexing at the given generation, for the given document type. */ public ApplicationReindexing withPending(String cluster, String documentType, long requiredGeneration) { - Cluster current = clusters.getOrDefault(cluster, Cluster.empty); + Cluster current = clusters.getOrDefault(cluster, Cluster.ready(common)); Cluster modified = new Cluster(current.common, with(documentType, requirePositive(requiredGeneration), current.pending), without(documentType, current.ready)); @@ -79,13 +79,10 @@ public class ApplicationReindexing implements Reindexing { if (clusters.get(cluster).pending().containsKey(documentType)) return Optional.empty(); - Status documentStatus = clusters.get(cluster).ready().get(documentType); - Status clusterStatus = clusters.get(cluster).common(); - if (documentStatus == null || documentStatus.ready().isBefore(clusterStatus.ready())) - documentStatus = clusterStatus; + if (clusters.get(cluster).ready().containsKey(documentType)) + return Optional.of(clusters.get(cluster).ready().get(documentType)); - if (documentStatus.ready().isAfter(common().ready())) - return Optional.of(documentStatus); + return Optional.of(clusters.get(cluster).common()); } return Optional.of(common()); } @@ -116,7 +113,7 @@ public class ApplicationReindexing implements Reindexing { /** Reindexing status for a single content cluster in an application. */ public static class Cluster { - private static final Cluster empty = new Cluster(Status.ALWAYS_READY, Map.of(), Map.of()); + private static Cluster ready(Status common) { return new Cluster(common, Map.of(), Map.of()); } private final Status common; private final Map<String, Long> pending; @@ -173,9 +170,6 @@ public class ApplicationReindexing implements Reindexing { /** Reindexing status common to an application, one of its clusters, or a single document type in a cluster. */ public static class Status implements Reindexing.Status { - /** Always ready, i.e., ignored when joining with more specific statuses. */ - private static final Status ALWAYS_READY = new Status(Instant.EPOCH); - private final Instant ready; Status(Instant ready) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 9b82cafcb49..8b9f2f084e1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -248,7 +248,7 @@ public class SessionPreparer { void preprocess() { try { this.preprocessedApplicationPackage = applicationPackage.preprocess(properties.zone(), logger); - } catch (IOException | TransformerException | ParserConfigurationException | SAXException e) { + } catch (IOException | TransformerException | ParserConfigurationException | SAXException | RuntimeException e) { throw new IllegalArgumentException("Error preprocessing application package for " + applicationId, e); } checkTimeout("preprocess"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationReindexingTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationReindexingTest.java index b803413d6ea..82de3e2eefb 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationReindexingTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationReindexingTest.java @@ -27,12 +27,19 @@ public class ApplicationReindexingTest { .withReady("one", "a", Instant.ofEpochMilli(1)) .withReady("two", "c", Instant.ofEpochMilli(3)); - assertEquals(Instant.ofEpochMilli(1 << 20), + // Document is most specific, and is used. + assertEquals(Instant.ofEpochMilli(1), reindexing.status("one", "a").orElseThrow().ready()); + // Cluster is most specific, and inherits application's common status. assertEquals(Instant.ofEpochMilli(1 << 20), reindexing.status("one", "d").orElseThrow().ready()); + // Cluster is most specific, and has its own status set. + assertEquals(Instant.ofEpochMilli(2 << 10), + reindexing.status("two", "d").orElseThrow().ready()); + + // Application is most specific, as cluster and documeent are unknown. assertEquals(Instant.ofEpochMilli(1 << 20), reindexing.status("three", "a").orElseThrow().ready()); @@ -45,7 +52,7 @@ public class ApplicationReindexingTest { assertEquals(Set.of("one", "two"), reindexing.clusters().keySet()); - assertEquals(new Status(Instant.EPOCH), + assertEquals(new Status(Instant.ofEpochMilli(1 << 20)), reindexing.clusters().get("one").common()); assertEquals(Map.of("a", new Status(Instant.ofEpochMilli(1))), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java index f4a553e25b7..57582040552 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainerTest.java @@ -16,13 +16,14 @@ public class ReindexingMaintainerTest { @Test public void testReadyComputation() { - ApplicationReindexing reindexing = ApplicationReindexing.ready(Instant.ofEpochMilli(1 << 20)) + ApplicationReindexing reindexing = ApplicationReindexing.ready(Instant.EPOCH) .withPending("one", "a", 10) .withReady("two", "b", Instant.ofEpochMilli(2)) .withPending("two", "b", 20) .withReady("two", Instant.ofEpochMilli(2 << 10)) .withReady("one", "a", Instant.ofEpochMilli(1)) - .withReady("two", "c", Instant.ofEpochMilli(3)); + .withReady("two", "c", Instant.ofEpochMilli(3)) + .withReady(Instant.ofEpochMilli(1 << 20)); assertEquals(reindexing, withReady(reindexing, () -> -1L, Instant.EPOCH)); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java index 2f05c99ab66..ce03dccc86b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java @@ -52,7 +52,7 @@ public interface BillingController { void deleteBillingInfo(TenantName tenant, Set<User> users, boolean isPrivileged); default CollectionMethod getCollectionMethod(TenantName tenant) { - return CollectionMethod.AUTO; + return CollectionMethod.NONE; } default CollectionResult setCollectionMethod(TenantName tenant, CollectionMethod method) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CollectionMethod.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CollectionMethod.java index 010ce56d1d7..fd586b0faf0 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CollectionMethod.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CollectionMethod.java @@ -1,6 +1,8 @@ package com.yahoo.vespa.hosted.controller.api.integration.billing; public enum CollectionMethod { - AUTO, - INVOICE + NONE, + EPAY, + INVOICE, + AUTO // Deprecated - this has never been serialized and can be removed in subsequent release } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CollectionResult.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CollectionResult.java index d2eb77ba94c..4322cdc81b3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CollectionResult.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/CollectionResult.java @@ -1,6 +1,7 @@ // Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.billing; +import java.util.Objects; import java.util.Optional; /** @@ -31,4 +32,24 @@ public class CollectionResult { public Optional<String> getErrorMessage() { return errorMessage; } + + @Override + public String toString() { + return "CollectionResult{" + + "errorMessage=" + errorMessage + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CollectionResult that = (CollectionResult) o; + return Objects.equals(errorMessage, that.errorMessage); + } + + @Override + public int hashCode() { + return Objects.hash(errorMessage); + } }
\ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java index 4b09c744537..b84c3083f41 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java @@ -24,7 +24,7 @@ public class MockBillingController implements BillingController { Map<TenantName, List<Invoice>> committedInvoices = new HashMap<>(); Map<TenantName, Invoice> uncommittedInvoices = new HashMap<>(); Map<TenantName, List<Invoice.LineItem>> unusedLineItems = new HashMap<>(); - CollectionMethod collectionMethod = CollectionMethod.AUTO; + Map<TenantName, CollectionMethod> collectionMethod = new HashMap<>(); @Override public PlanId getPlan(TenantName tenant) { @@ -138,12 +138,12 @@ public class MockBillingController implements BillingController { @Override public CollectionMethod getCollectionMethod(TenantName tenant) { - return collectionMethod; + return collectionMethod.getOrDefault(tenant, CollectionMethod.AUTO); } @Override public CollectionResult setCollectionMethod(TenantName tenant, CollectionMethod method) { - collectionMethod = method; + collectionMethod.put(tenant, method); return CollectionResult.success(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java index cf135fc6b6e..de57f1a5676 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculator.java @@ -26,7 +26,7 @@ public class DeploymentQuotaCalculator { DeploymentSpec deploymentSpec) { if (tenantQuota.budget().isEmpty()) return tenantQuota; // Shortcut if there is no budget limit to care about. - if (deployingZone.environment().isTest()) return Quota.unlimited(); + if (deployingZone.environment().isTest()) return tenantQuota; if (deployingZone.environment().isProduction()) return probablyEnoughForAll(tenantQuota, tenantApps, deployingApp, deploymentSpec); return getMaximumAllowedQuota(tenantQuota, tenantApps, deployingApp, deployingZone); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index 0dce968960f..dfef5b85e40 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -51,7 +51,6 @@ public class BillingApiHandler extends LoggingRequestHandler { private static final String OPTIONAL_PREFIX = "/api"; private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - private final BillingController billingController; private final ApplicationController applicationController; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java index 98710ce9b26..4099de32e00 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/DeploymentQuotaCalculatorTest.java @@ -64,8 +64,9 @@ public class DeploymentQuotaCalculatorTest { } @Test - public void unlimited_quota_in_pipeline() { - Quota calculated = DeploymentQuotaCalculator.calculate(Quota.zero(), List.of(), ApplicationId.defaultId(), ZoneId.from("test", "apac1"), DeploymentSpec.empty); - assertEquals(Quota.unlimited(), calculated); + public void tenant_quota_in_pipeline() { + var tenantQuota = Quota.unlimited().withBudget(42); + var calculated = DeploymentQuotaCalculator.calculate(tenantQuota, List.of(), ApplicationId.defaultId(), ZoneId.from("test", "apac1"), DeploymentSpec.empty); + assertEquals(tenantQuota, calculated); } }
\ No newline at end of file diff --git a/document/src/vespa/document/base/globalid.cpp b/document/src/vespa/document/base/globalid.cpp index bdf81d1c3c8..4f96ac0e659 100644 --- a/document/src/vespa/document/base/globalid.cpp +++ b/document/src/vespa/document/base/globalid.cpp @@ -39,39 +39,6 @@ getHexVal(char c) namespace document { -bool -GlobalId::BucketOrderCmp::operator()(const GlobalId &lhs, const GlobalId &rhs) const -{ - const unsigned char * __restrict__ a = lhs._gid._buffer; - const unsigned char * __restrict__ b = rhs._gid._buffer; - int diff; - if ((diff = compare(a[0], b[0])) != 0) { - return diff < 0; - } - if ((diff = compare(a[1], b[1])) != 0) { - return diff < 0; - } - if ((diff = compare(a[2], b[2])) != 0) { - return diff < 0; - } - if ((diff = compare(a[3], b[3])) != 0) { - return diff < 0; - } - if ((diff = compare(a[8], b[8])) != 0) { - return diff < 0; - } - if ((diff = compare(a[9], b[9])) != 0) { - return diff < 0; - } - if ((diff = compare(a[10], b[10])) != 0) { - return diff < 0; - } - if ((diff = compare(a[11], b[11])) != 0) { - return diff < 0; - } - return lhs < rhs; -} - vespalib::string GlobalId::toString() const { vespalib::asciistream out; out << "gid(0x" << vespalib::hex; diff --git a/document/src/vespa/document/base/globalid.h b/document/src/vespa/document/base/globalid.h index 1bb257b7cbf..ac30e189033 100644 --- a/document/src/vespa/document/base/globalid.h +++ b/document/src/vespa/document/base/globalid.h @@ -64,7 +64,24 @@ public: * given bucket. */ struct BucketOrderCmp { - bool operator()(const GlobalId &lhs, const GlobalId &rhs) const; + bool operator()(const GlobalId &lhs, const GlobalId &rhs) const { + const uint32_t * __restrict__ a = lhs._gid._nums; + const uint32_t * __restrict__ b = rhs._gid._nums; + if (a[0] != b[0]) { + return bitswap(a[0]) < bitswap(b[0]); + } + if (a[2] != b[2]) { + return bitswap(a[2]) < bitswap(b[2]); + } + return __builtin_bswap32(a[1]) < __builtin_bswap32(b[1]); + } + static uint32_t bitswap(uint32_t value) { + value = ((value & 0x55555555) << 1) | ((value & 0xaaaaaaaa) >> 1); + value = ((value & 0x33333333) << 2) | ((value & 0xcccccccc) >> 2); + value = ((value & 0x0f0f0f0f) << 4) | ((value & 0xf0f0f0f0) >> 4); + return __builtin_bswap32(value); + } + //These 2 compare methods are exposed only for testing static int compareRaw(unsigned char a, unsigned char b) { return a - b; |