diff options
| author | Harald Musum <musum@verizonmedia.com> | 2020-04-27 21:23:07 +0200 |
|---|---|---|
| committer | Harald Musum <musum@verizonmedia.com> | 2020-04-27 21:23:07 +0200 |
| commit | 380bb7be6070516c0ef0e5b7963632680d45e93c (patch) | |
| tree | 60fbd8d6a88ae130af49d49f313fc6590a52a4c6 | |
| parent | d68b6c23b59ca1292df8b85e82301d4e8516297e (diff) | |
| parent | 737fb3450c5bb253aec2c1ca8959093885593421 (diff) | |
Merge branch 'master' into hmusum/use-docker-image-instead-of-string-3
649 files changed, 5912 insertions, 4269 deletions
diff --git a/application-preprocessor/pom.xml b/application-preprocessor/pom.xml index b3b85a776fe..613a2747c35 100644 --- a/application-preprocessor/pom.xml +++ b/application-preprocessor/pom.xml @@ -59,11 +59,6 @@ <version>${project.version}</version> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> diff --git a/athenz-identity-provider-service/pom.xml b/athenz-identity-provider-service/pom.xml index f15ebe87b03..daedc1d0b9b 100644 --- a/athenz-identity-provider-service/pom.xml +++ b/athenz-identity-provider-service/pom.xml @@ -39,12 +39,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-core</artifactId> <scope>provided</scope> diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java index 7dc3496adc9..ae4a5933ac2 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice; import com.google.inject.Inject; import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyUtils; @@ -148,7 +148,7 @@ public class ConfigserverSslContextFactoryProvider extends TlsContextBasedProvid SiaUtils.writePrivateKeyFile(VESPA_SIA_DIRECTORY, configserverIdentity, privateKey); Instant expirationTime = certificate.getNotAfter().toInstant(); Duration expiry = Duration.between(certificate.getNotBefore().toInstant(), expirationTime); - log.log(LogLevel.INFO, String.format("Got Athenz x509 certificate with expiry %s (expires %s)", expiry, expirationTime)); + log.log(Level.INFO, String.format("Got Athenz x509 certificate with expiry %s (expires %s)", expiry, expirationTime)); return KeyStoreBuilder.withType(KeyStoreType.JKS) .withKeyEntry(CERTIFICATE_ALIAS, privateKey, keystorePwd, certificate) .build(); @@ -168,13 +168,13 @@ public class ConfigserverSslContextFactoryProvider extends TlsContextBasedProvid @Override public void run() { try { - log.log(LogLevel.INFO, "Updating configserver provider certificate from ZTS"); + log.log(Level.INFO, "Updating configserver provider certificate from ZTS"); char[] keystorePwd = generateKeystorePassword(); KeyStore keyStore = updateKeystore(configserverIdentity, keystorePwd, keyProvider, ztsClient, athenzProviderServiceConfig); keyManager.updateKeystore(keyStore, keystorePwd); - log.log(LogLevel.INFO, "Certificate successfully updated"); + log.log(Level.INFO, "Certificate successfully updated"); } catch (Throwable t) { - log.log(LogLevel.ERROR, "Failed to update certificate from ZTS: " + t.getMessage(), t); + log.log(Level.SEVERE, "Failed to update certificate from ZTS: " + t.getMessage(), t); } } } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java index 7151de9ccc9..cb0b1e0557f 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/identitydocument/IdentityDocumentResource.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice.identitydocument; import com.google.inject.Inject; import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.IdentityType; import com.yahoo.vespa.athenz.identityprovider.api.bindings.IdentityDocumentApi; @@ -43,7 +43,7 @@ public class IdentityDocumentResource implements IdentityDocumentApi { return EntityBindingsMapper.toSignedIdentityDocumentEntity(identityDocumentGenerator.generateSignedIdentityDocument(hostname, identityType)); } catch (Exception e) { String message = String.format("Unable to generate identity doument for '%s': %s", hostname, e.getMessage()); - log.log(LogLevel.ERROR, message, e); + log.log(Level.SEVERE, message, e); throw new InternalServerErrorException(message, e); } } diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java index 7a4e82d134b..dbe26d3a6bb 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceConfirmationResource.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmati import com.google.inject.Inject; import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import javax.ws.rs.Consumes; import javax.ws.rs.ForbiddenException; @@ -32,9 +32,9 @@ public class InstanceConfirmationResource { @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public InstanceConfirmation confirmInstance(InstanceConfirmation instanceConfirmation) { - log.log(LogLevel.DEBUG, instanceConfirmation.toString()); + log.log(Level.FINE, instanceConfirmation.toString()); if (!instanceValidator.isValidInstance(instanceConfirmation)) { - log.log(LogLevel.ERROR, "Invalid instance: " + instanceConfirmation); + log.log(Level.SEVERE, "Invalid instance: " + instanceConfirmation); throw new ForbiddenException("Instance is invalid"); } return instanceConfirmation; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java index 53972a39c97..b49e03658e6 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceRefreshResource.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.athenz.instanceproviderservice.instanceconfirmati import com.google.inject.Inject; import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import javax.ws.rs.Consumes; import javax.ws.rs.ForbiddenException; @@ -34,9 +34,9 @@ public class InstanceRefreshResource { @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public InstanceConfirmation confirmInstanceRefresh(InstanceConfirmation instanceConfirmation) { - log.log(LogLevel.DEBUG, instanceConfirmation.toString()); + log.log(Level.FINE, instanceConfirmation.toString()); if (!instanceValidator.isValidRefresh(instanceConfirmation)) { - log.log(LogLevel.ERROR, "Invalid instance refresh: " + instanceConfirmation); + log.log(Level.SEVERE, "Invalid instance refresh: " + instanceConfirmation); throw new ForbiddenException("Instance is invalid"); } return instanceConfirmation; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java index 3a926922f2b..05bbd790ad7 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/instanceconfirmation/InstanceValidator.java @@ -7,7 +7,7 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identityprovider.api.EntityBindingsMapper; import com.yahoo.vespa.athenz.identityprovider.api.SignedIdentityDocument; @@ -77,7 +77,7 @@ public class InstanceValidator { VespaUniqueInstanceId csrProviderUniqueId = getVespaUniqueInstanceId(instanceConfirmation); if(! providerUniqueId.equals(csrProviderUniqueId)) { - log.log(LogLevel.WARNING, String.format("Instance %s has invalid provider unique ID in CSR (%s)", providerUniqueId, csrProviderUniqueId)); + log.log(Level.WARNING, String.format("Instance %s has invalid provider unique ID in CSR (%s)", providerUniqueId, csrProviderUniqueId)); return false; } @@ -85,14 +85,14 @@ public class InstanceValidator { return false; } - log.log(LogLevel.DEBUG, () -> String.format("Validating instance %s.", providerUniqueId)); + log.log(Level.FINE, () -> String.format("Validating instance %s.", providerUniqueId)); PublicKey publicKey = keyProvider.getPublicKey(signedIdentityDocument.signingKeyVersion()); if (signer.hasValidSignature(signedIdentityDocument, publicKey)) { - log.log(LogLevel.DEBUG, () -> String.format("Instance %s is valid.", providerUniqueId)); + log.log(Level.FINE, () -> String.format("Instance %s is valid.", providerUniqueId)); return true; } - log.log(LogLevel.ERROR, () -> String.format("Instance %s has invalid signature.", providerUniqueId)); + log.log(Level.SEVERE, () -> String.format("Instance %s has invalid signature.", providerUniqueId)); return false; } @@ -100,14 +100,14 @@ public class InstanceValidator { // We'll have to perform some validation on the instance id and other fields of the attribute map. // Separate between tenant and node certificate as well. public boolean isValidRefresh(InstanceConfirmation confirmation) { - log.log(LogLevel.DEBUG, () -> String.format("Accepting refresh for instance with identity '%s', provider '%s', instanceId '%s'.", + log.log(Level.FINE, () -> String.format("Accepting refresh for instance with identity '%s', provider '%s', instanceId '%s'.", new AthenzService(confirmation.domain, confirmation.service).getFullName(), confirmation.provider, confirmation.attributes.get(SAN_DNS_ATTRNAME))); try { return validateAttributes(confirmation, getVespaUniqueInstanceId(confirmation)); } catch (Exception e) { - log.log(LogLevel.WARNING, "Encountered exception while refreshing certificate for confirmation: " + confirmation, e); + log.log(Level.WARNING, "Encountered exception while refreshing certificate for confirmation: " + confirmation, e); return false; } } @@ -131,7 +131,7 @@ public class InstanceValidator { private boolean validateAttributes(InstanceConfirmation confirmation, VespaUniqueInstanceId vespaUniqueInstanceId) { if(vespaUniqueInstanceId == null) { - log.log(LogLevel.WARNING, "Unable to find unique instance ID in refresh request: " + confirmation.toString()); + log.log(Level.WARNING, "Unable to find unique instance ID in refresh request: " + confirmation.toString()); return false; } @@ -142,7 +142,7 @@ public class InstanceValidator { .findFirst() // Should be only one .orElse(null); if(node == null) { - log.log(LogLevel.WARNING, "Invalid InstanceConfirmation, No nodes matching uniqueId: " + vespaUniqueInstanceId); + log.log(Level.WARNING, "Invalid InstanceConfirmation, No nodes matching uniqueId: " + vespaUniqueInstanceId); return false; } @@ -162,7 +162,7 @@ public class InstanceValidator { // Validate that ipaddresses in request are valid for node if(! nodeIpAddresses.containsAll(ips)) { - log.log(LogLevel.WARNING, "Invalid InstanceConfirmation, wrong ip in : " + vespaUniqueInstanceId); + log.log(Level.WARNING, "Invalid InstanceConfirmation, wrong ip in : " + vespaUniqueInstanceId); return false; } return true; diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java index 466a9783bb3..0325971038f 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/ca/restapi/CertificateAuthorityApiHandler.java @@ -7,7 +7,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.http.servlet.ServletRequest; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.SlimeJsonResponse; @@ -101,7 +101,7 @@ public class CertificateAuthorityApiHandler extends LoggingRequestHandler { confirmation.set(InstanceValidator.SAN_IPS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRegistration.csr(), SubjectAlternativeName.Type.IP_ADDRESS)); confirmation.set(InstanceValidator.SAN_DNS_ATTRNAME, Certificates.getSubjectAlternativeNames(instanceRegistration.csr(), SubjectAlternativeName.Type.DNS_NAME)); if (!instanceValidator.isValidInstance(confirmation)) { - log.log(LogLevel.INFO, "Invalid instance registration for " + instanceRegistration.toString()); + log.log(Level.INFO, "Invalid instance registration for " + instanceRegistration.toString()); return ErrorResponse.forbidden("Unable to launch service: " +instanceRegistration.service()); } var certificate = certificates.create(instanceRegistration.csr(), caCertificate(), caPrivateKey()); diff --git a/build_settings.cmake b/build_settings.cmake index ac4e86c2e81..c96a626ad0b 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -54,7 +54,12 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" ST endif() else() set(CXX_SPECIFIC_WARN_OPTS "-Wsuggest-override -Wnon-virtual-dtor -Wformat-security") - set(VESPA_ATOMIC_LIB "atomic") + if(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8" OR + VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.1") + set(VESPA_ATOMIC_LIB "") + else() + set(VESPA_ATOMIC_LIB "atomic") + endif() set(VESPA_GCC_LIB "gcc") set(VESPA_STDCXX_FS_LIB "stdc++fs") endif() @@ -79,7 +84,11 @@ if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -latomic -ldl") endif() else() - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--build-id -latomic -ldl -Wl,-E") + if(VESPA_ATOMIC_LIB STREQUAL "") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--build-id -ldl -Wl,-E") + else() + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--build-id -latomic -ldl -Wl,-E") + endif() endif() SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic" ) diff --git a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java index 7f0acd83b0d..67aef138135 100644 --- a/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java +++ b/clustercontroller-apps/src/main/java/com/yahoo/vespa/clustercontroller/apps/clustercontroller/StateRestApiV2Handler.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.clustercontroller.apps.clustercontroller; import com.google.inject.Inject; import com.yahoo.cloud.config.ClusterInfoConfig; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.apputil.communication.http.JDiscHttpRequestHandler; import com.yahoo.vespa.clustercontroller.core.restapiv2.ClusterControllerStateRestAPI; import com.yahoo.vespa.clustercontroller.utils.staterestapi.server.RestApiHandler; @@ -44,7 +44,7 @@ public class StateRestApiV2Handler extends JDiscHttpRequestHandler { } if (result.isEmpty()) { log.warning("Found no cluster controller in model config"); - } else if (log.isLoggable(LogLevel.DEBUG)) { + } else if (log.isLoggable(Level.FINE)) { StringBuilder sb = new StringBuilder(); sb.append("Found ").append(result.size()).append(" cluster controllers in model config:"); for (Map.Entry<Integer, ClusterControllerStateRestAPI.Socket> e : result.entrySet()) { diff --git a/clustercontroller-core/pom.xml b/clustercontroller-core/pom.xml index a952016e3bf..ee503d91540 100644 --- a/clustercontroller-core/pom.xml +++ b/clustercontroller-core/pom.xml @@ -19,6 +19,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespalog</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> @@ -60,7 +66,7 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - <dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>zkfacade</artifactId> <version>${project.version}</version> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java index 62cd158c759..eefa7dfb322 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateView.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; @@ -102,7 +102,7 @@ public class ClusterStateView { // The distributor may be old (null), or the distributor may not have updated // to the latest state version just yet. We log here with fine, because it may // also be a symptom of something wrong. - log.log(LogLevel.DEBUG, "Current state version is " + currentStateVersion + + log.log(Level.FINE, "Current state version is " + currentStateVersion + ", while host info received from distributor " + node.getNodeIndex() + " is " + hostVersion); return; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java index 964e7c5e947..367771ff87a 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/EventLog.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter; @@ -57,9 +57,9 @@ public class EventLog implements EventLogInterface { } if (e instanceof NodeEvent) { - addNodeOnlyEvent((NodeEvent)e, logInfo ? LogLevel.INFO: LogLevel.DEBUG); + addNodeOnlyEvent((NodeEvent)e, logInfo ? Level.INFO: Level.FINE); } else { - log.log(logInfo ? LogLevel.INFO : LogLevel.DEBUG, e.toString()); + log.log(logInfo ? Level.INFO : Level.FINE, e.toString()); } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java index d943cf27f9c..658dd10f7e5 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/FleetController.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.document.FixedBucketSpaces; import com.yahoo.exception.ExceptionUtils; import com.yahoo.jrt.ListenFailedException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; @@ -299,12 +299,12 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd public void shutdown() throws InterruptedException, java.io.IOException { if (runner != null && isRunning()) { - log.log(LogLevel.INFO, "Joining event thread."); + log.log(Level.INFO, "Joining event thread."); running.set(false); synchronized(monitor) { monitor.notifyAll(); } runner.join(); } - log.log(LogLevel.INFO, "Fleetcontroller done shutting down event thread."); + log.log(Level.INFO, "Fleetcontroller done shutting down event thread."); controllerThreadId = Thread.currentThread().getId(); database.shutdown(this); @@ -321,7 +321,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd public void updateOptions(FleetControllerOptions options, long configGeneration) { synchronized(monitor) { assert(this.options.fleetControllerIndex == options.fleetControllerIndex); - log.log(LogLevel.INFO, "Fleetcontroller " + options.fleetControllerIndex + " has new options"); + log.log(Level.INFO, "Fleetcontroller " + options.fleetControllerIndex + " has new options"); nextOptions = options.clone(); nextConfigGeneration = configGeneration; monitor.notifyAll(); @@ -416,7 +416,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd */ public void handleFleetData(Map<Integer, Integer> data) { verifyInControllerThread(); - log.log(LogLevel.SPAM, "Sending fleet data event on to master election handler"); + log.log(Level.FINEST, "Sending fleet data event on to master election handler"); metricUpdater.updateMasterElectionMetrics(data); masterElectionHandler.handleFleetData(data); } @@ -510,9 +510,9 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd try{ rpcServer.setSlobrokConnectionSpecs(options.slobrokConnectionSpecs, options.rpcPort); } catch (ListenFailedException e) { - log.log(LogLevel.WARNING, "Failed to bind RPC server to port " + options.rpcPort +". This may be natural if cluster has altered the services running on this node: " + e.getMessage()); + log.log(Level.WARNING, "Failed to bind RPC server to port " + options.rpcPort +". This may be natural if cluster has altered the services running on this node: " + e.getMessage()); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to initialize RPC server socket: " + e.getMessage()); + log.log(Level.WARNING, "Failed to initialize RPC server socket: " + e.getMessage()); } } @@ -520,7 +520,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd try{ statusPageServer.setPort(options.httpPort); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to initialize status server socket. This may be natural if cluster has altered the services running on this node: " + e.getMessage()); + log.log(Level.WARNING, "Failed to initialize status server socket. This may be natural if cluster has altered the services running on this node: " + e.getMessage()); } } @@ -548,7 +548,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd responseCode = StatusPageResponse.ResponseCode.INTERNAL_SERVER_ERROR; message = "Internal Server Error"; hiddenMessage = ExceptionUtils.getStackTraceAsString(e);; - log.log(LogLevel.DEBUG, "Unknown exception thrown for request " + httpRequest.getRequest() + + log.log(Level.FINE, "Unknown exception thrown for request " + httpRequest.getRequest() + ": " + hiddenMessage); } @@ -633,7 +633,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } catch (InterruptedException e) { throw (InterruptedException) new InterruptedException("Interrupted").initCause(e); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to watch master election: " + e.toString()); + log.log(Level.WARNING, "Failed to watch master election: " + e.toString()); } return false; } @@ -652,7 +652,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd try { propagateOptions(); } catch (Exception e) { - log.log(LogLevel.ERROR, "Failed to handle new fleet controller config", e); + log.log(Level.SEVERE, "Failed to handle new fleet controller config", e); } } @@ -671,7 +671,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd // If there's a pending DB store we have not yet been able to store the // current state bundle to ZK and must therefore _not_ allow it to be published. if (database.hasPendingClusterStateMetaDataStore()) { - log.log(LogLevel.DEBUG, "Can't publish current cluster state as it has one or more pending ZooKeeper stores"); + log.log(Level.FINE, "Can't publish current cluster state as it has one or more pending ZooKeeper stores"); return false; } boolean sentAny = false; @@ -682,7 +682,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd && currentTime >= nextStateSendTime) { if (currentTime < firstAllowedStateBroadcast) { - log.log(LogLevel.DEBUG, "Not set to broadcast states just yet, but as we have gotten info from all nodes we can do so safely."); + log.log(Level.FINE, "Not set to broadcast states just yet, but as we have gotten info from all nodes we can do so safely."); // Reset timer to only see warning once. firstAllowedStateBroadcast = currentTime; } @@ -811,7 +811,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd taskCompletionQueue.remove(); } else if (taskCompletion.getDeadlineTimePointMs() <= now) { var details = buildNodesNotYetConvergedMessage(taskCompletion.getMinimumVersion()); - log.log(LogLevel.WARNING, () -> String.format("Deferred task of type '%s' has exceeded wait deadline; completing with failure (details: %s)", + log.log(Level.WARNING, () -> String.format("Deferred task of type '%s' has exceeded wait deadline; completing with failure (details: %s)", taskCompletion.getTask().getClass().getName(), details)); taskCompletion.getTask().handleFailure(RemoteClusterControllerTask.Failure.of( RemoteClusterControllerTask.FailureCondition.DEADLINE_EXCEEDED, details)); @@ -1037,7 +1037,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd + stateVersionTracker.getCurrentVersion() + " to be in line.", timer.getCurrentTimeInMillis())); long currentTime = timer.getCurrentTimeInMillis(); firstAllowedStateBroadcast = currentTime + options.minTimeBeforeFirstSystemStateBroadcast; - log.log(LogLevel.DEBUG, "At time " + currentTime + " we set first system state broadcast time to be " + log.log(Level.FINE, "At time " + currentTime + " we set first system state broadcast time to be " + options.minTimeBeforeFirstSystemStateBroadcast + " ms after at time " + firstAllowedStateBroadcast + "."); didWork = true; } @@ -1067,10 +1067,10 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd tick(); } } catch (InterruptedException e) { - log.log(LogLevel.DEBUG, "Event thread stopped by interrupt exception: " + e); + log.log(Level.FINE, "Event thread stopped by interrupt exception: " + e); } catch (Throwable t) { t.printStackTrace(); - log.log(LogLevel.ERROR, "Fatal error killed fleet controller", t); + log.log(Level.SEVERE, "Fatal error killed fleet controller", t); synchronized (monitor) { running.set(false); } System.exit(1); } finally { @@ -1125,7 +1125,7 @@ public class FleetController implements NodeStateOrHostInfoChangeHandler, NodeAd } } if (ackedNodes >= nodeCount) { - log.log(LogLevel.INFO, ackedNodes + " nodes now have acked system state " + version + " or higher."); + log.log(Level.INFO, ackedNodes + " nodes now have acked system state " + version + " or higher."); return; } long remainingTime = maxTime - System.currentTimeMillis(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java index 9d23eead605..7b026dd6a74 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MasterElectionHandler.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; import java.util.Map; @@ -34,7 +34,7 @@ public class MasterElectionHandler implements MasterInterface { this.nextInLineCount = Integer.MAX_VALUE; // Only a given set of nodes can ever become master if (index > (totalCount - 1) / 2) { - log.log(LogLevel.DEBUG, "Cluster controller " + index + ": We can never become master and will always stay a follower."); + log.log(Level.FINE, "Cluster controller " + index + ": We can never become master and will always stay a follower."); } // Tag current time as when we have not seen any other master. Make sure we're not taking over at once for master that is on the way down masterGoneFromZooKeeperTime = timer.getCurrentTimeInMillis(); @@ -109,32 +109,32 @@ public class MasterElectionHandler implements MasterInterface { if (totalCount == 1) return false; // No point in doing master election with only one node configured to be cluster controller if (nextMasterData == null) { if (masterCandidate == null) { - log.log(LogLevel.SPAM, "Cluster controller " + index + ": No current master candidate. Waiting for data to do master election."); + log.log(Level.FINEST, "Cluster controller " + index + ": No current master candidate. Waiting for data to do master election."); } return false; // Nothing have happened since last time. } // Move next data to temporary, such that we don't need to keep lock, and such that we don't retry // if we happen to fail processing the data. Map<Integer, Integer> state; - log.log(LogLevel.INFO, "Cluster controller " + index + ": Handling new master election, as we have received " + nextMasterData.size() + " entries"); + log.log(Level.INFO, "Cluster controller " + index + ": Handling new master election, as we have received " + nextMasterData.size() + " entries"); synchronized (monitor) { state = nextMasterData; nextMasterData = null; } - log.log(LogLevel.INFO, "Cluster controller " + index + ": Got master election state " + toString(state) + "."); + log.log(Level.INFO, "Cluster controller " + index + ": Got master election state " + toString(state) + "."); if (state.isEmpty()) throw new IllegalStateException("Database has no master data. We should at least have data for ourselves."); Map.Entry<Integer, Integer> first = state.entrySet().iterator().next(); Integer currentMaster = getMaster(); if (currentMaster != null && first.getKey().intValue() != currentMaster.intValue()) { - log.log(LogLevel.INFO, "Cluster controller " + index + ": Master gone from ZooKeeper. Tagging timestamp. Will wait " + this.masterZooKeeperCooldownPeriod + " ms."); + log.log(Level.INFO, "Cluster controller " + index + ": Master gone from ZooKeeper. Tagging timestamp. Will wait " + this.masterZooKeeperCooldownPeriod + " ms."); masterGoneFromZooKeeperTime = timer.getCurrentTimeInMillis(); masterCandidate = null; } if (first.getValue().intValue() != first.getKey().intValue()) { - log.log(LogLevel.INFO, "Fleet controller " + index + ": First index is not currently trying to become master. Waiting for it to change state"); + log.log(Level.INFO, "Fleet controller " + index + ": First index is not currently trying to become master. Waiting for it to change state"); masterCandidate = null; if (first.getKey() == index) { - log.log(LogLevel.INFO, "Cluster controller " + index + ": We are next in line to become master. Altering our state to look for followers"); + log.log(Level.INFO, "Cluster controller " + index + ": We are next in line to become master. Altering our state to look for followers"); database.setMasterVote(dbContext, index); } } else { @@ -148,21 +148,21 @@ public class MasterElectionHandler implements MasterInterface { if (2 * followers > totalCount) { Integer newMaster = getMaster(); if (newMaster != null && currentMaster != null && newMaster.intValue() == currentMaster.intValue()) { - log.log(LogLevel.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + currentMaster + " is still the master"); + log.log(Level.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + currentMaster + " is still the master"); } else if (newMaster != null && currentMaster != null) { - log.log(LogLevel.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + newMaster + " took over for fleet controller " + currentMaster + " as master"); + log.log(Level.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + newMaster + " took over for fleet controller " + currentMaster + " as master"); } else if (newMaster == null) { - log.log(LogLevel.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + masterCandidate + " is new master candidate, but needs to wait before it can take over"); + log.log(Level.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + masterCandidate + " is new master candidate, but needs to wait before it can take over"); } else { - log.log(LogLevel.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + newMaster + " is newly elected master"); + log.log(Level.INFO, "MASTER_ELECTION: Cluster controller " + index + ": " + newMaster + " is newly elected master"); } } else { - log.log(LogLevel.INFO, "MASTER_ELECTION: Cluster controller " + index + ": Currently too few followers for cluster controller candidate " + masterCandidate + ". No current master. (" + followers + "/" + totalCount + " followers)"); + log.log(Level.INFO, "MASTER_ELECTION: Cluster controller " + index + ": Currently too few followers for cluster controller candidate " + masterCandidate + ". No current master. (" + followers + "/" + totalCount + " followers)"); } Integer ourState = state.get(index); if (ourState == null) throw new IllegalStateException("Database lacks data from ourselves. This should always be present."); if (ourState.intValue() != first.getKey().intValue()) { - log.log(LogLevel.INFO, "Cluster controller " + index + ": Altering our state to follow new fleet controller master candidate " + first.getKey()); + log.log(Level.INFO, "Cluster controller " + index + ": Altering our state to follow new fleet controller master candidate " + first.getKey()); database.setMasterVote(dbContext, first.getKey()); } } @@ -179,7 +179,7 @@ public class MasterElectionHandler implements MasterInterface { if (nextInLineCount != ourPosition) { nextInLineCount = ourPosition; if (ourPosition > 0) { - log.log(LogLevel.DEBUG, "Cluster controller " + index + ": We are now " + getPosition(nextInLineCount) + " in queue to take over being master."); + log.log(Level.FINE, "Cluster controller " + index + ": We are now " + getPosition(nextInLineCount) + " in queue to take over being master."); } } } @@ -209,7 +209,7 @@ public class MasterElectionHandler implements MasterInterface { } public void handleFleetData(Map<Integer, Integer> data) { - log.log(LogLevel.INFO, "Cluster controller " + index + ": Got new fleet data with " + data.size() + " entries: " + data); + log.log(Level.INFO, "Cluster controller " + index + ": Got new fleet data with " + data.size() + " entries: " + data); synchronized (monitor) { nextMasterData = data; monitor.notifyAll(); @@ -218,7 +218,7 @@ public class MasterElectionHandler implements MasterInterface { public void lostDatabaseConnection() { if (totalCount > 1) { - log.log(LogLevel.INFO, "Cluster controller " + index + ": Clearing master data as we lost connection on node " + index); + log.log(Level.INFO, "Cluster controller " + index + ": Clearing master data as we lost connection on node " + index); masterData = null; masterCandidate = null; followers = 0; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java index 82d13e2d9ef..652c3039430 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.collections.Pair; import com.yahoo.jrt.Target; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.distribution.Group; import com.yahoo.vdslib.state.*; @@ -167,7 +167,7 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { } if (prematureCrashCount != count) { prematureCrashCount = count; - log.log(LogLevel.DEBUG, "Premature crash count on " + toString() + " set to " + count); + log.log(Level.FINE, "Premature crash count on " + toString() + " set to " + count); } } public int getPrematureCrashCount() { return prematureCrashCount; } @@ -309,17 +309,17 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { if (state.getState().oneOf("dsm") && !reportedState.getState().oneOf("dsm")) { wentDownWithStartTime = reportedState.getStartTimestamp(); wentDownAtClusterState = getNewestSystemStateSent(); - log.log(LogLevel.DEBUG, "Setting going down timestamp of node " + node + " to " + wentDownWithStartTime); + log.log(Level.FINE, "Setting going down timestamp of node " + node + " to " + wentDownWithStartTime); } if (state.getState().equals(State.DOWN) && !reportedState.getState().oneOf("d")) { downStableStateTime = time; - log.log(LogLevel.DEBUG, "Down stable state on " + toString() + " altered to " + time); + log.log(Level.FINE, "Down stable state on " + toString() + " altered to " + time); if (reportedState.getState() == State.INITIALIZING) { recentlyObservedUnstableDuringInit = true; } } else if (state.getState().equals(State.UP) && !reportedState.getState().oneOf("u")) { upStableStateTime = time; - log.log(LogLevel.DEBUG, "Up stable state on " + toString() + " altered to " + time); + log.log(Level.FINE, "Up stable state on " + toString() + " altered to " + time); } if (!state.getState().validReportedNodeState(node.getType())) { throw new IllegalStateException("Trying to set illegal reported node state: " + state); @@ -342,19 +342,19 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { } else { nextAttemptTime = time + 5000; } - log.log(LogLevel.SPAM, "Failed to get state from node " + toString() + ", scheduling next attempt in " + (nextAttemptTime - time) + " ms."); + log.log(Level.FINEST, "Failed to get state from node " + toString() + ", scheduling next attempt in " + (nextAttemptTime - time) + " ms."); } else { connectionAttemptCount = 0; timeOfFirstFailingConnectionAttempt = 0; reportedState = state; if (version == 0 || state.getState().equals(State.STOPPING)) { nextAttemptTime = time + cluster.getPollingFrequency(); - log.log(LogLevel.SPAM, "Scheduling next attempt to get state from " + toString() + " in " + (nextAttemptTime - time) + " ms (polling freq)."); + log.log(Level.FINEST, "Scheduling next attempt to get state from " + toString() + " in " + (nextAttemptTime - time) + " ms (polling freq)."); } else { nextAttemptTime = time; } } - log.log(LogLevel.SPAM, "Set reported state of node " + this + " to " + reportedState + ". Next connection attempt is at " + nextAttemptTime); + log.log(Level.FINEST, "Set reported state of node " + this + " to " + reportedState + ". Next connection attempt is at " + nextAttemptTime); } /** Sets the wanted state. The wanted state is taken as UP if a null argument is given */ @@ -374,7 +374,7 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { } } wantedState = newWanted; - log.log(LogLevel.SPAM, "Set wanted state of node " + this + " to " + wantedState + "."); + log.log(Level.FINEST, "Set wanted state of node " + this + " to " + wantedState + "."); } public long getTimeForNextStateRequestAttempt() { @@ -388,16 +388,16 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { downgradeToRpcVersion(RPCCommunicator.LEGACY_SET_SYSTEM_STATE2_RPC_VERSION, methodName, timer); return true; } else if (timer.getCurrentTimeInMillis() - 2000 < adjustedVersionTime) { - log.log(LogLevel.DEBUG, () -> "Node " + toString() + " does not support " + methodName + " call. Version already downgraded, so ignoring it."); + log.log(Level.FINE, () -> "Node " + toString() + " does not support " + methodName + " call. Version already downgraded, so ignoring it."); return true; } } - log.log(LogLevel.WARNING, "Node " + toString() + " does not support " + methodName + " which it should."); + log.log(Level.WARNING, "Node " + toString() + " does not support " + methodName + " which it should."); return false; } private void downgradeToRpcVersion(int newVersion, String methodName, Timer timer) { - log.log(LogLevel.DEBUG, () -> String.format("Node %s does not support %s call. Downgrading to version %d.", + log.log(Level.FINE, () -> String.format("Node %s does not support %s call. Downgrading to version %d.", toString(), methodName, newVersion)); version = newVersion; nextAttemptTime = 0; @@ -453,7 +453,7 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { && (wentDownAtClusterState == null || wentDownAtClusterState.getVersion() < stateBundle.getVersion()) && !stateBundle.getBaselineClusterState().getNodeState(node).getState().oneOf("dsm")) { - log.log(LogLevel.DEBUG, () -> String.format("Clearing going down timestamp of node %s after " + + log.log(Level.FINE, () -> String.format("Clearing going down timestamp of node %s after " + "receiving ack of cluster state bundle %s", node, stateBundle)); wentDownWithStartTime = 0; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java index 318768454c6..da32caa0c4e 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.jrt.ErrorCode; import com.yahoo.jrt.Target; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; @@ -64,12 +64,12 @@ public class NodeStateGatherer { if (info.getTimeForNextStateRequestAttempt() > currentTime) continue; // too early if (info.getRpcAddress() == null || info.isRpcAddressOutdated()) { // Cannot query state of node without RPC address - log.log(LogLevel.DEBUG, "Not sending getNodeState request to node " + info.getNode() + ": Not in slobrok"); + log.log(Level.FINE, "Not sending getNodeState request to node " + info.getNode() + ": Not in slobrok"); NodeState reportedState = info.getReportedState().clone(); if (( ! reportedState.getState().equals(State.DOWN) && currentTime - info.getRpcAddressOutdatedTimestamp() > maxSlobrokDisconnectGracePeriod) || reportedState.getState().equals(State.STOPPING)) // Don't wait for grace period if we expect node to be stopping { - log.log(LogLevel.DEBUG, "Setting reported state to DOWN " + log.log(Level.FINE, "Setting reported state to DOWN " + (reportedState.getState().equals(State.STOPPING) ? "as node completed stopping." : "as node has been out of slobrok longer than " + maxSlobrokDisconnectGracePeriod + ".")); @@ -102,7 +102,7 @@ public class NodeStateGatherer { NodeInfo info = req.getNodeInfo(); if (!info.isPendingGetNodeStateRequest(req)) { - log.log(LogLevel.DEBUG, "Ignoring getnodestate response from " + info.getNode() + log.log(Level.FINE, "Ignoring getnodestate response from " + info.getNode() + " as request replied to is not the most recent pending request."); continue; } @@ -118,10 +118,10 @@ public class NodeStateGatherer { listener.handleNewNodeState(info, newState.clone()); info.setReportedState(newState, currentTime); } else { - log.log(LogLevel.DEBUG, "Ignoring get node state error. Need to resend"); + log.log(Level.FINE, "Ignoring get node state error. Need to resend"); } } else { - log.log(LogLevel.DEBUG, "Ignoring getnodestate response from " + info.getNode() + " as it was aborted by client"); + log.log(Level.FINE, "Ignoring getnodestate response from " + info.getNode() + " as it was aborted by client"); } continue; @@ -135,7 +135,7 @@ public class NodeStateGatherer { listener.handleNewNodeState(info, state.clone()); info.setReportedState(state, currentTime); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to process get node state response", e); + log.log(Level.WARNING, "Failed to process get node state response", e); info.setReportedState(new NodeState(info.getNode().getType(), State.DOWN), currentTime); } @@ -157,9 +157,9 @@ public class NodeStateGatherer { if (req.getReply().getReturnCode() == ErrorCode.TIMEOUT) { String msg = "RPC timeout"; if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + "RPC timeout talking to node.", NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + "RPC timeout talking to node.", NodeEvent.Type.REPORTED, currentTime), Level.INFO); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setDescription(msg); } else if (req.getReply().getReturnCode() == ErrorCode.CONNECTION) { @@ -172,9 +172,9 @@ public class NodeStateGatherer { if (msg.equals("Connection refused")) { msg = "Connection error: Connection refused"; if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.INFO); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setState(State.DOWN); @@ -184,24 +184,24 @@ public class NodeStateGatherer { msg += " Node is no longer in slobrok."; } if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.INFO); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setState(State.DOWN).setDescription(msg); } else if (msg.equals("Connection timed out")) { if (info.getReportedState().getState().oneOf("ui")) { msg = "Connection error: Timeout"; - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.INFO); } else { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } } else { msg = "Connection error: " + reason; if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.WARNING); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.WARNING); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setDescription(msg); } @@ -210,9 +210,9 @@ public class NodeStateGatherer { req.getReply().getReturnCode() + ": " + req.getReply().getReturnMessage(); if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.WARNING); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.WARNING); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setDescription(msg); } @@ -221,30 +221,30 @@ public class NodeStateGatherer { } else if (req.getReply().getReturnCode() == ErrorCode.NO_SUCH_METHOD) { String msg = "no such RPC method error"; if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.WARNING); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.WARNING); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setState(State.DOWN).setDescription(msg + ": get node state"); } else if (req.getReply().getReturnCode() == 75004) { String msg = "Node refused to answer RPC request and is likely stopping: " + req.getReply().getReturnMessage(); // The node is shutting down and is not accepting requests from anyone if (info.getReportedState().getState().equals(State.STOPPING)) { - log.log(LogLevel.DEBUG, "Failed to get node state from " + info + " because it is still shutting down."); + log.log(Level.FINE, "Failed to get node state from " + info + " because it is still shutting down."); } else { if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.INFO); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } } newState.setState(State.STOPPING).setDescription(msg); } else { String msg = "Got unexpected error, assumed to be node issue " + req.getReply().getReturnCode() + ": " + req.getReply().getReturnMessage(); if (info.getReportedState().getState().oneOf("ui")) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), LogLevel.WARNING); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(info, prefix + msg, NodeEvent.Type.REPORTED, currentTime), Level.WARNING); } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { - log.log(LogLevel.DEBUG, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); + log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } newState.setState(State.DOWN).setDescription(msg); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java index 3c19f70d1e2..cb6bacf97a6 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.jrt.Spec; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.state.*; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; @@ -51,7 +51,7 @@ public class StateChangeHandler { final DatabaseHandler database, final DatabaseHandler.Context dbContext) throws InterruptedException { int startTimestampsReset = 0; - log.log(LogLevel.DEBUG, String.format("handleAllDistributorsInSync invoked for state version %d", currentState.getVersion())); + log.log(Level.FINE, String.format("handleAllDistributorsInSync invoked for state version %d", currentState.getVersion())); for (NodeType nodeType : NodeType.getTypes()) { for (ConfiguredNode configuredNode : nodes) { final Node node = new Node(nodeType, configuredNode.index()); @@ -59,20 +59,20 @@ public class StateChangeHandler { final NodeState nodeState = currentState.getNodeState(node); if (nodeInfo != null && nodeState != null) { if (nodeState.getStartTimestamp() > nodeInfo.getStartTimestamp()) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, String.format("Storing away new start timestamp for node %s (%d)", + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, String.format("Storing away new start timestamp for node %s (%d)", node, nodeState.getStartTimestamp())); } nodeInfo.setStartTimestamp(nodeState.getStartTimestamp()); } if (nodeState.getStartTimestamp() > 0) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, String.format("Resetting timestamp in cluster state for node %s", node)); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, String.format("Resetting timestamp in cluster state for node %s", node)); } ++startTimestampsReset; } - } else if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, node + ": " + + } else if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, node + ": " + (nodeInfo == null ? "null" : nodeInfo.getStartTimestamp()) + ", " + (nodeState == null ? "null" : nodeState.getStartTimestamp())); } @@ -85,7 +85,7 @@ public class StateChangeHandler { stateMayHaveChanged = true; database.saveStartTimestamps(dbContext); } else { - log.log(LogLevel.DEBUG, "Found no start timestamps to reset in cluster state."); + log.log(Level.FINE, "Found no start timestamps to reset in cluster state."); } } @@ -118,7 +118,7 @@ public class StateChangeHandler { final NodeStateOrHostInfoChangeHandler nodeListener) { final NodeState currentState = currentClusterState.getNodeState(node.getNode()); - final LogLevel level = (currentState.equals(reportedState) && node.getVersion() == 0) ? LogLevel.SPAM : LogLevel.DEBUG; + final Level level = (currentState.equals(reportedState) && node.getVersion() == 0) ? Level.FINEST : Level.FINE; if (log.isLoggable(level)) { log.log(level, String.format("Got nodestate reply from %s: %s (Current state is %s)", node, node.getReportedState().getTextualDifference(reportedState), currentState.toString(true))); @@ -132,9 +132,9 @@ public class StateChangeHandler { // *** LOGGING ONLY if ( ! reportedState.similarTo(node.getReportedState())) { if (reportedState.getState().equals(State.DOWN)) { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(node, "Failed to get node state: " + reportedState.toString(true), NodeEvent.Type.REPORTED, currentTime), LogLevel.INFO); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(node, "Failed to get node state: " + reportedState.toString(true), NodeEvent.Type.REPORTED, currentTime), Level.INFO); } else { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(node, "Now reporting state " + reportedState.toString(true), NodeEvent.Type.REPORTED, currentTime), LogLevel.DEBUG); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(node, "Now reporting state " + reportedState.toString(true), NodeEvent.Type.REPORTED, currentTime), Level.FINE); } } @@ -147,13 +147,13 @@ public class StateChangeHandler { if (reportedState.getMinUsedBits() != currentState.getMinUsedBits()) { final int oldCount = currentState.getMinUsedBits(); final int newCount = reportedState.getMinUsedBits(); - log.log(LogLevel.DEBUG, + log.log(Level.FINE, String.format("Altering node state to reflect that min distribution bit count has changed from %d to %d", oldCount, newCount)); eventLog.add(NodeEvent.forBaseline(node, String.format("Altered min distribution bit count from %d to %d", oldCount, newCount), NodeEvent.Type.CURRENT, currentTime), isMaster); - } else if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, String.format("Not altering state of %s in cluster state because new state is too similar: %s", + } else if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, String.format("Not altering state of %s in cluster state because new state is too similar: %s", node, currentState.getTextualDifference(reportedState))); } @@ -181,12 +181,12 @@ public class StateChangeHandler { } if (node.getReportedState().getState().equals(State.STOPPING)) { - log.log(LogLevel.DEBUG, "Node " + node.getNode() + " is no longer in slobrok. Was in stopping state, so assuming it has shut down normally. Setting node down"); + log.log(Level.FINE, "Node " + node.getNode() + " is no longer in slobrok. Was in stopping state, so assuming it has shut down normally. Setting node down"); NodeState ns = node.getReportedState().clone(); ns.setState(State.DOWN); handleNewReportedNodeState(currentClusterState, node, ns.clone(), nodeListener); } else { - log.log(LogLevel.DEBUG, "Node " + node.getNode() + " no longer in slobrok was in state " + node.getReportedState() + ". Waiting to see if it reappears in slobrok"); + log.log(Level.FINE, "Node " + node.getNode() + " no longer in slobrok was in state " + node.getReportedState() + ". Waiting to see if it reappears in slobrok"); } stateMayHaveChanged = true; @@ -208,8 +208,8 @@ public class StateChangeHandler { } stateMayHaveChanged = true; - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, String.format("Got new wanted nodestate for %s: %s", node, currentState.getTextualDifference(proposedState))); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, String.format("Got new wanted nodestate for %s: %s", node, currentState.getTextualDifference(proposedState))); } // Should be checked earlier before state was set in cluster assert(proposedState.getState().validWantedNodeState(node.getNode().getType())); @@ -321,11 +321,11 @@ public class StateChangeHandler { if (mayResetCrashCounterOnStableUpNode(currentTime, node, lastReportedState)) { node.setPrematureCrashCount(0); - log.log(LogLevel.DEBUG, "Resetting premature crash count on node " + node + " as it has been up for a long time."); + log.log(Level.FINE, "Resetting premature crash count on node " + node + " as it has been up for a long time."); triggeredAnyTimers = true; } else if (mayResetCrashCounterOnStableDownNode(currentTime, node, lastReportedState)) { node.setPrematureCrashCount(0); - log.log(LogLevel.DEBUG, "Resetting premature crash count on node " + node + " as it has been down for a long time."); + log.log(Level.FINE, "Resetting premature crash count on node " + node + " as it has been down for a long time."); triggeredAnyTimers = true; } @@ -415,8 +415,8 @@ public class StateChangeHandler { final NodeState reportedState, final NodeStateOrHostInfoChangeHandler nodeListener) { final long timeNow = timer.getCurrentTimeInMillis(); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, String.format("Finding new cluster state entry for %s switching state %s", + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, String.format("Finding new cluster state entry for %s switching state %s", node, currentState.getTextualDifference(reportedState))); } @@ -425,8 +425,8 @@ public class StateChangeHandler { } if (initializationProgressHasIncreased(currentState, reportedState)) { node.setInitProgressTime(timeNow); - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Reset initialize timer on " + node + " to " + node.getInitProgressTime()); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Reset initialize timer on " + node + " to " + node.getInitProgressTime()); } } if (handleImplicitCrashEdgeFromReverseInitProgress(node, currentState, reportedState, nodeListener, timeNow)) { @@ -482,7 +482,7 @@ public class StateChangeHandler { if (nodeUpToDownEdge(node, currentState, reportedState)) { node.setTransitionTime(timeNow); if (node.getUpStableStateTime() + stableStateTimePeriod > timeNow && !isControlledShutdown(reportedState)) { - log.log(LogLevel.DEBUG, "Stable state: " + node.getUpStableStateTime() + " + " + stableStateTimePeriod + " > " + timeNow); + log.log(Level.FINE, "Stable state: " + node.getUpStableStateTime() + " + " + stableStateTimePeriod + " > " + timeNow); eventLog.add(NodeEvent.forBaseline(node, String.format("Stopped or possibly crashed after %d ms, which is before " + "stable state time period. Premature crash count is now %d.", diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java index 5ecb57a1c76..502a0823b13 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/SystemStateBroadcaster.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.jrt.ErrorCode; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.*; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; @@ -66,7 +66,7 @@ public class SystemStateBroadcaster { long time = timer.getCurrentTimeInMillis(); Long lastReported = lastErrorReported.get(info.getNode()); boolean alreadySeen = (lastReported != null && time - lastReported < minTimeBetweenNodeErrorLogging); - log.log((nodeOk && !alreadySeen) ? LogLevel.WARNING : LogLevel.DEBUG, message); + log.log((nodeOk && !alreadySeen) ? Level.WARNING : Level.FINE, message); if (!alreadySeen) { lastErrorReported.put(info.getNode(), time); } @@ -92,11 +92,11 @@ public class SystemStateBroadcaster { // NO_SUCH_METHOD implies node is on a version that does not understand explicit activations // and it has already merrily started using the state version. Treat as if it had been ACKed. if (reply.getReturnCode() != ErrorCode.NO_SUCH_METHOD) { - log.log(LogLevel.DEBUG, () -> String.format("Activation NACK for node %s with version %d, message %s", + log.log(Level.FINE, () -> String.format("Activation NACK for node %s with version %d, message %s", info, version, reply.getReturnMessage())); success = false; } else { - log.log(LogLevel.DEBUG, () -> String.format("Node %s did not understand state activation RPC; " + + log.log(Level.FINE, () -> String.format("Node %s did not understand state activation RPC; " + "implicitly treating state %d as activated on node", info, version)); } } else if (reply.getActualVersion() != version) { @@ -109,7 +109,7 @@ public class SystemStateBroadcaster { version, info, reply.getActualVersion())); success = false; } else { - log.log(LogLevel.DEBUG, () -> String.format("Node %s reports successful activation of state " + + log.log(Level.FINE, () -> String.format("Node %s reports successful activation of state " + "version %d", info, version)); } info.setSystemStateVersionActivationAcked(version, success); @@ -140,7 +140,7 @@ public class SystemStateBroadcaster { } } else { info.setClusterStateBundleVersionAcknowledged(version, true); - log.log(LogLevel.DEBUG, () -> String.format("Node %s ACKed system state version %d.", info, version)); + log.log(Level.FINE, () -> String.format("Node %s ACKed system state version %d.", info, version)); lastErrorReported.remove(info.getNode()); } } @@ -216,7 +216,7 @@ public class SystemStateBroadcaster { if (!anyDistributorsNeedStateBundle && (currentStateVersion > lastStateVersionBundleAcked)) { markCurrentClusterStateBundleAsReceivedByAllDistributors(); if (clusterStateBundle.deferredActivation()) { - log.log(LogLevel.DEBUG, () -> String.format("All distributors have ACKed cluster state " + + log.log(Level.FINE, () -> String.format("All distributors have ACKed cluster state " + "version %d, sending activation", currentStateVersion)); } else { markCurrentClusterStateAsConverged(database, dbContext, fleetController); @@ -235,7 +235,7 @@ public class SystemStateBroadcaster { if (!anyDistributorsNeedActivation && (currentStateVersion > lastClusterStateVersionConverged)) { markCurrentClusterStateAsConverged(database, dbContext, fleetController); } else { - log.log(LogLevel.DEBUG, () -> String.format("distributors still need activation in state %d (last converged: %d)", + log.log(Level.FINE, () -> String.format("distributors still need activation in state %d (last converged: %d)", currentStateVersion, lastClusterStateVersionConverged)); } } @@ -245,7 +245,7 @@ public class SystemStateBroadcaster { } private void markCurrentClusterStateAsConverged(DatabaseHandler database, DatabaseHandler.Context dbContext, FleetController fleetController) throws InterruptedException { - log.log(LogLevel.DEBUG, "All distributors have newest clusterstate, updating start timestamps in zookeeper and clearing them from cluster state"); + log.log(Level.FINE, "All distributors have newest clusterstate, updating start timestamps in zookeeper and clearing them from cluster state"); lastClusterStateVersionConverged = clusterStateBundle.getVersion(); lastClusterStateBundleConverged = clusterStateBundle; fleetController.handleAllDistributorsInSync(database, dbContext); @@ -263,7 +263,7 @@ public class SystemStateBroadcaster { ClusterState baselineState = clusterStateBundle.getBaselineClusterState(); if (!baselineState.isOfficial()) { - log.log(LogLevel.INFO, String.format("Publishing cluster state version %d", baselineState.getVersion())); + log.log(Level.INFO, String.format("Publishing cluster state version %d", baselineState.getVersion())); baselineState.setOfficial(true); // FIXME this violates state bundle immutability } @@ -272,11 +272,11 @@ public class SystemStateBroadcaster { if (nodeNeedsToObserveStartupTimestamps(node)) { // TODO this is the same for all nodes, compute only once ClusterStateBundle modifiedBundle = clusterStateBundle.cloneWithMapper(state -> buildModifiedClusterState(state, dbContext)); - log.log(LogLevel.DEBUG, () -> String.format("Sending modified cluster state version %d" + + log.log(Level.FINE, () -> String.format("Sending modified cluster state version %d" + " to node %s: %s", baselineState.getVersion(), node, modifiedBundle)); communicator.setSystemState(modifiedBundle, node, setClusterStateWaiter); } else { - log.log(LogLevel.DEBUG, () -> String.format("Sending system state version %d to node %s. " + + log.log(Level.FINE, () -> String.format("Sending system state version %d to node %s. " + "(went down time %d, node start time %d)", baselineState.getVersion(), node, node.getWentDownWithStartTime(), node.getStartTimestamp())); communicator.setSystemState(clusterStateBundle, node, setClusterStateWaiter); @@ -297,7 +297,7 @@ public class SystemStateBroadcaster { var recipients = resolveStateActivationSendSet(dbContext); for (NodeInfo node : recipients) { - log.log(LogLevel.DEBUG, () -> String.format("Sending cluster state activation to node %s for version %d", + log.log(Level.FINE, () -> String.format("Sending cluster state activation to node %s for version %d", node, clusterStateBundle.getVersion())); communicator.activateClusterStateVersion(clusterStateBundle.getVersion(), node, activateClusterStateVersionWaiter); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java index b9d02f16090..bb0692c2e24 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/DatabaseHandler.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.database; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.Node; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; @@ -52,7 +52,7 @@ public class DatabaseHandler { } private class DatabaseListener implements Database.DatabaseListener { public void handleZooKeeperSessionDown() { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Lost contact with zookeeper server"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Lost contact with zookeeper server"); synchronized(monitor) { lostZooKeeperConnectionEvent = true; monitor.notifyAll(); @@ -62,7 +62,7 @@ public class DatabaseHandler { public void handleMasterData(Map<Integer, Integer> data) { synchronized (monitor) { if (masterDataEvent != null && masterDataEvent.equals(data)) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": New master data was the same as the last one. Not responding to it"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": New master data was the same as the last one. Not responding to it"); } else { masterDataEvent = data; } @@ -115,7 +115,7 @@ public class DatabaseHandler { synchronized (databaseMonitor) { wasRunning = database != null; if (wasRunning) { - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Resetting database state"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Resetting database state"); database.close(); database = null; } @@ -123,7 +123,7 @@ public class DatabaseHandler { clearSessionMetaData(); if (wasRunning) { - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Done resetting database state"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Done resetting database state"); } } @@ -133,7 +133,7 @@ public class DatabaseHandler { currentlyStored.clear(); pendingStore.clear(); pendingStore.masterVote = currentVote; - log.log(LogLevel.DEBUG, "Cleared session metadata. Pending master vote is now " + log.log(Level.FINE, "Cleared session metadata. Pending master vote is now " + pendingStore.masterVote); } @@ -141,7 +141,7 @@ public class DatabaseHandler { if (address == null && zooKeeperAddress == null) return; if (address != null && zooKeeperAddress != null && address.equals(zooKeeperAddress)) return; if (zooKeeperAddress != null) { - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": " + (address == null ? "Stopped using ZooKeeper." : "Got new ZooKeeper address to use: " + address)); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": " + (address == null ? "Stopped using ZooKeeper." : "Got new ZooKeeper address to use: " + address)); } zooKeeperAddress = address; reset(); @@ -149,7 +149,7 @@ public class DatabaseHandler { public void setZooKeeperSessionTimeout(int timeout) { if (timeout == zooKeeperSessionTimeout) return; - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Got new ZooKeeper session timeout of " + timeout + " milliseconds."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Got new ZooKeeper session timeout of " + timeout + " milliseconds."); zooKeeperSessionTimeout = timeout; reset(); } @@ -165,7 +165,7 @@ public class DatabaseHandler { } // We still hold the database lock while calling this, we want to block callers. clearSessionMetaData(); - log.log(LogLevel.INFO, + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Setting up new ZooKeeper session at " + zooKeeperAddress); DatabaseFactory.Params params = new DatabaseFactory.Params() .cluster(cluster) @@ -176,20 +176,20 @@ public class DatabaseHandler { database = databaseFactory.create(params); } } catch (KeeperException.NodeExistsException e) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Cannot create ephemeral fleetcontroller node. ZooKeeper server " + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Cannot create ephemeral fleetcontroller node. ZooKeeper server " + "not seen old fleetcontroller instance disappear? It already exists. Will retry later: " + e.getMessage()); } catch (InterruptedException e) { throw (InterruptedException) new InterruptedException("Interrupted").initCause(e); } catch (KeeperException.ConnectionLossException e) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to connect to ZooKeeper at " + zooKeeperAddress + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to connect to ZooKeeper at " + zooKeeperAddress + " with session timeout " + zooKeeperSessionTimeout + ": " + e.getMessage()); } catch (Exception e) { StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to connect to ZooKeeper at " + zooKeeperAddress + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to connect to ZooKeeper at " + zooKeeperAddress + " with session timeout " + zooKeeperSessionTimeout + ": " + sw); } - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Done setting up new ZooKeeper session at " + zooKeeperAddress); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Done setting up new ZooKeeper session at " + zooKeeperAddress); } /** @@ -202,17 +202,17 @@ public class DatabaseHandler { synchronized (monitor) { if (zooKeeperAddress == null) return false; // If not using zookeeper no work to be done if (lostZooKeeperConnectionEvent) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": doNextZooKeeperTask(): lost connection"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": doNextZooKeeperTask(): lost connection"); context.getFleetController().lostDatabaseConnection(); lostZooKeeperConnectionEvent = false; didWork = true; if (masterDataEvent != null) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Had new master data queued on disconnect. Removing master data event"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Had new master data queued on disconnect. Removing master data event"); masterDataEvent = null; } } if (masterDataEvent != null) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": doNextZooKeeperTask(): new master data"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": doNextZooKeeperTask(): new master data"); if (!masterDataEvent.containsKey(nodeIndex)) { Integer currentVote = (pendingStore.masterVote != null ? pendingStore.masterVote : currentlyStored.masterVote); assert(currentVote != null); @@ -239,7 +239,7 @@ public class DatabaseHandler { didWork |= performZooKeeperWrites(); } } catch (CasWriteFailed e) { - log.log(LogLevel.WARNING, String.format("CaS write to ZooKeeper failed, another controller " + + log.log(Level.WARNING, String.format("CaS write to ZooKeeper failed, another controller " + "has likely taken over ownership: %s", e.getMessage())); // Clear DB and master election state. This shall trigger a full re-fetch of all // version and election-related metadata. @@ -257,21 +257,21 @@ public class DatabaseHandler { boolean didWork = false; if (pendingStore.masterVote != null) { didWork = true; - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Attempting to store master vote " + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Attempting to store master vote " + pendingStore.masterVote + " into zookeeper."); if (database.storeMasterVote(pendingStore.masterVote)) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Managed to store master vote " + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Managed to store master vote " + pendingStore.masterVote + " into zookeeper."); currentlyStored.masterVote = pendingStore.masterVote; pendingStore.masterVote = null; } else { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to store master vote"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to store master vote"); return true; } } if (pendingStore.lastSystemStateVersion != null) { didWork = true; - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Attempting to store last system state version " + pendingStore.lastSystemStateVersion + " into zookeeper."); if (database.storeLatestSystemStateVersion(pendingStore.lastSystemStateVersion)) { @@ -283,7 +283,7 @@ public class DatabaseHandler { } if (pendingStore.startTimestamps != null) { didWork = true; - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Attempting to store " + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Attempting to store " + pendingStore.startTimestamps.size() + " start timestamps into zookeeper."); if (database.storeStartTimestamps(pendingStore.startTimestamps)) { currentlyStored.startTimestamps = pendingStore.startTimestamps; @@ -294,7 +294,7 @@ public class DatabaseHandler { } if (pendingStore.wantedStates != null) { didWork = true; - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Attempting to store " + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Attempting to store " + pendingStore.wantedStates.size() + " wanted states into zookeeper."); if (database.storeWantedStates(pendingStore.wantedStates)) { currentlyStored.wantedStates = pendingStore.wantedStates; @@ -316,7 +316,7 @@ public class DatabaseHandler { } public void setMasterVote(Context context, int wantedMasterCandidate) throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Checking if master vote has been updated and need to be stored."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Checking if master vote has been updated and need to be stored."); // Schedule a write if one of the following is true: // - There is already a pending vote to be written, that may have been written already without our knowledge // - We don't know what is actually stored now @@ -324,14 +324,14 @@ public class DatabaseHandler { if (pendingStore.masterVote != null || currentlyStored.masterVote == null || currentlyStored.masterVote != wantedMasterCandidate) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Scheduling master vote " + wantedMasterCandidate + " to be stored in zookeeper."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Scheduling master vote " + wantedMasterCandidate + " to be stored in zookeeper."); pendingStore.masterVote = wantedMasterCandidate; doNextZooKeeperTask(context); } } public void saveLatestSystemStateVersion(Context context, int version) throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Checking if latest system state version has been updated and need to be stored."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Checking if latest system state version has been updated and need to be stored."); // Schedule a write if one of the following is true: // - There is already a pending vote to be written, that may have been written already without our knowledge // - We don't know what is actually stored now @@ -339,14 +339,14 @@ public class DatabaseHandler { if (pendingStore.lastSystemStateVersion != null || currentlyStored.lastSystemStateVersion == null || currentlyStored.lastSystemStateVersion != version) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Scheduling new last system state version " + version + " to be stored in zookeeper."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Scheduling new last system state version " + version + " to be stored in zookeeper."); pendingStore.lastSystemStateVersion = version; doNextZooKeeperTask(context); } } public int getLatestSystemStateVersion() throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Retrieving latest system state version."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Retrieving latest system state version."); synchronized (databaseMonitor) { if (database != null && !database.isClosed()) { currentlyStored.lastSystemStateVersion = database.retrieveLatestSystemStateVersion(); @@ -355,7 +355,7 @@ public class DatabaseHandler { Integer version = currentlyStored.lastSystemStateVersion; if (version == null) { if (usingZooKeeper()) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to retrieve latest system state version from ZooKeeper. Returning version 0."); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to retrieve latest system state version from ZooKeeper. Returning version 0."); } return 0; // FIXME "fail-oblivious" is not a good error handling mode for such a critical component! } @@ -363,7 +363,7 @@ public class DatabaseHandler { } public void saveLatestClusterStateBundle(Context context, ClusterStateBundle clusterStateBundle) throws InterruptedException { - log.log(LogLevel.DEBUG, () -> String.format("Fleetcontroller %d: Scheduling bundle %s to be saved to ZooKeeper", nodeIndex, clusterStateBundle)); + log.log(Level.FINE, () -> String.format("Fleetcontroller %d: Scheduling bundle %s to be saved to ZooKeeper", nodeIndex, clusterStateBundle)); pendingStore.clusterStateBundle = clusterStateBundle; doNextZooKeeperTask(context); } @@ -378,7 +378,7 @@ public class DatabaseHandler { } public ClusterStateBundle getLatestClusterStateBundle() throws InterruptedException { - log.log(LogLevel.DEBUG, () -> String.format("Fleetcontroller %d: Retrieving latest cluster state bundle from ZooKeeper", nodeIndex)); + log.log(Level.FINE, () -> String.format("Fleetcontroller %d: Retrieving latest cluster state bundle from ZooKeeper", nodeIndex)); synchronized (databaseMonitor) { if (database != null && !database.isClosed()) { return database.retrieveLastPublishedStateBundle(); @@ -389,7 +389,7 @@ public class DatabaseHandler { } public void saveWantedStates(Context context) throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Checking whether wanted states have changed compared to zookeeper version."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Checking whether wanted states have changed compared to zookeeper version."); Map<Node, NodeState> wantedStates = new TreeMap<>(); for (NodeInfo info : context.getCluster().getNodeInfo()) { if (!info.getUserWantedState().equals(new NodeState(info.getNode().getType(), State.UP))) { @@ -403,14 +403,14 @@ public class DatabaseHandler { if (pendingStore.wantedStates != null || currentlyStored.wantedStates == null || !currentlyStored.wantedStates.equals(wantedStates)) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Scheduling new wanted states to be stored into zookeeper."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Scheduling new wanted states to be stored into zookeeper."); pendingStore.wantedStates = wantedStates; doNextZooKeeperTask(context); } } public boolean loadWantedStates(Context context) throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Retrieving node wanted states."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Retrieving node wanted states."); synchronized (databaseMonitor) { if (database != null && !database.isClosed()) { currentlyStored.wantedStates = database.retrieveWantedStates(); @@ -421,7 +421,7 @@ public class DatabaseHandler { if (usingZooKeeper()) { // We get here if the ZooKeeper client has lost the connection to ZooKeeper. // TODO: Should instead fail the tick until connected!? - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Failed to retrieve wanted states from ZooKeeper. Assuming UP for all nodes."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Failed to retrieve wanted states from ZooKeeper. Assuming UP for all nodes."); } wantedStates = new TreeMap<>(); } @@ -435,7 +435,7 @@ public class DatabaseHandler { context.getNodeStateUpdateListener().handleNewWantedNodeState(nodeInfo, wantedState); altered = true; } - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Node " + node + " has wanted state " + wantedState); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Node " + node + " has wanted state " + wantedState); } // Remove wanted state from any node having a wanted state set that is no longer valid @@ -451,13 +451,13 @@ public class DatabaseHandler { } public void saveStartTimestamps(Context context) throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Scheduling start timestamps to be stored into zookeeper."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Scheduling start timestamps to be stored into zookeeper."); pendingStore.startTimestamps = context.getCluster().getStartTimestamps(); doNextZooKeeperTask(context); } public boolean loadStartTimestamps(ContentCluster cluster) throws InterruptedException { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Retrieving start timestamps"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Retrieving start timestamps"); synchronized (databaseMonitor) { if (database == null || database.isClosed()) { return false; @@ -467,13 +467,13 @@ public class DatabaseHandler { Map<Node, Long> startTimestamps = currentlyStored.startTimestamps; if (startTimestamps == null) { if (usingZooKeeper()) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to retrieve start timestamps from ZooKeeper. Cluster state will be bloated with timestamps until we get them set."); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Failed to retrieve start timestamps from ZooKeeper. Cluster state will be bloated with timestamps until we get them set."); } startTimestamps = new TreeMap<>(); } for (Map.Entry<Node, Long> e : startTimestamps.entrySet()) { cluster.setStartTimestamp(e.getKey(), e.getValue()); - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Node " + e.getKey() + " has start timestamp " + e.getValue()); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Node " + e.getKey() + " has start timestamp " + e.getValue()); } return true; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java index 3ad0587b740..cd5202c89b2 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/MasterDataGatherer.java @@ -8,7 +8,7 @@ import java.util.logging.Logger; import java.util.*; import java.nio.charset.Charset; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; public class MasterDataGatherer { @@ -51,11 +51,11 @@ public class MasterDataGatherer { public void process(WatchedEvent watchedEvent) { switch (watchedEvent.getType()) { case NodeChildrenChanged: // Fleetcontrollers have either connected or disconnected to ZooKeeper - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": A change occurred in the list of registered fleetcontrollers. Requesting new information"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": A change occurred in the list of registered fleetcontrollers. Requesting new information"); session.getChildren(zooKeeperRoot + "indexes", this, childListener, null); break; case NodeDataChanged: // A fleetcontroller has changed what node it is voting for - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Altered data in node " + watchedEvent.getPath() + ". Requesting new vote"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Altered data in node " + watchedEvent.getPath() + ". Requesting new vote"); int index = getIndex(watchedEvent.getPath()); synchronized (nextMasterData) { nextMasterData.put(index, null); @@ -63,15 +63,15 @@ public class MasterDataGatherer { session.getData(zooKeeperRoot + "indexes/" + index, this, nodeListener, null); break; case NodeCreated: // How can this happen? Can one leave watches on non-existing nodes? - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeCreated"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeCreated"); break; case NodeDeleted: // We get this event when fleetcontrollers shut down and node in dir disappears. But it should also trigger a NodeChildrenChanged event, so // ignoring this one. - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Node deleted event gotten. Ignoring it, expecting a NodeChildrenChanged event too."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Node deleted event gotten. Ignoring it, expecting a NodeChildrenChanged event too."); break; case None: - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Got ZooKeeper event None."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Got ZooKeeper event None."); } } } @@ -84,13 +84,13 @@ public class MasterDataGatherer { private class DirCallback implements AsyncCallback.ChildrenCallback { public void processResult(int version, String path, Object context, List<String> nodes) { if (nodes == null) nodes = new LinkedList<String>(); - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Got node list response from " + path + " version " + version + " with " + nodes.size() + " nodes"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Got node list response from " + path + " version " + version + " with " + nodes.size() + " nodes"); synchronized (nextMasterData) { nextMasterData.clear(); for (String node : nodes) { int index = Integer.parseInt(node); nextMasterData.put(index, null); - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Attempting to fetch data in node '" + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Attempting to fetch data in node '" + zooKeeperRoot + index + "' to see vote"); session.getData(zooKeeperRoot + "indexes/" + index, changeWatcher, nodeListener, null); // Invocation of cycleCompleted() for fully accumulated election state will happen @@ -105,45 +105,45 @@ public class MasterDataGatherer { public void processResult(int code, String path, Object context, byte[] rawdata, Stat stat) { String data = rawdata == null ? null : new String(rawdata, utf8); - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Got vote data from path " + path + + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Got vote data from path " + path + " with code " + code + " and data " + data); int index = getIndex(path); synchronized (nextMasterData) { if (code != KeeperException.Code.OK.intValue()) { if (code == KeeperException.Code.NONODE.intValue()) { - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Node at " + path + + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Node at " + path + " removed, got no other option than counting it as down."); } else { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Failure code " + code + + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Failure code " + code + " when listening to node at " + path + ", will assume it's down."); } if (nextMasterData.containsKey(index)) { nextMasterData.remove(index); } else { - log.log(LogLevel.ERROR, "Fleetcontroller " + nodeIndex + ": Strangely, we already had data from node " + index + " when trying to remove it"); + log.log(Level.SEVERE, "Fleetcontroller " + nodeIndex + ": Strangely, we already had data from node " + index + " when trying to remove it"); } } else { Integer value = Integer.valueOf(data); if (nextMasterData.containsKey(index)) { if (value.equals(nextMasterData.get(index))) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + ", which already was " + value + "."); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + ", which already was " + value + "."); } else { - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + ". Altering vote from " + nextMasterData.get(index) + " to " + value + "."); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + ". Altering vote from " + nextMasterData.get(index) + " to " + value + "."); nextMasterData.put(index, value); } } else { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + " which is not alive according to current state. Ignoring it"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got vote from fleetcontroller " + index + " which is not alive according to current state. Ignoring it"); } } for(Integer vote : nextMasterData.values()) { if (vote == null) { - log.log(LogLevel.SPAM, "Fleetcontroller " + nodeIndex + ": Still not received votes from all fleet controllers. Awaiting more responses."); + log.log(Level.FINEST, "Fleetcontroller " + nodeIndex + ": Still not received votes from all fleet controllers. Awaiting more responses."); return; } } } - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Got votes for all fleetcontrollers. Sending event with new fleet data for update"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Got votes for all fleetcontrollers. Sending event with new fleet data for update"); cycleCompleted(); } } @@ -173,14 +173,14 @@ public class MasterDataGatherer { Map<Integer, Integer> copy; synchronized (nextMasterData) { if (nextMasterData.equals(masterData)) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": No change in master data detected, not sending it on"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": No change in master data detected, not sending it on"); // for(Integer i : nextMasterData.keySet()) { System.err.println(i + " -> " + nextMasterData.get(i)); } return; } masterData = new TreeMap<Integer, Integer>(nextMasterData); copy = masterData; } - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Got new master data, sending it on"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Got new master data, sending it on"); listener.handleMasterData(copy); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java index 880f2cdaf19..bba417b1333 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/database/ZooKeeperDatabase.java @@ -9,7 +9,7 @@ import com.yahoo.vespa.clustercontroller.core.rpc.SlimeClusterStateBundleCodec; import org.apache.zookeeper.*; import org.apache.zookeeper.data.Stat; import org.apache.zookeeper.data.ACL; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.State; import com.yahoo.vdslib.state.Node; @@ -52,43 +52,43 @@ public class ZooKeeperDatabase extends Database { public void process(WatchedEvent watchedEvent) { // Shouldn't get events after we expire, but just be sure we stop them here. if (state != null && state.equals(Event.KeeperState.Expired)) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got event from ZooKeeper session after it expired"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got event from ZooKeeper session after it expired"); return; } Event.KeeperState newState = watchedEvent.getState(); if (state == null || !state.equals(newState)) switch (newState) { case Expired: - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Zookeeper session expired"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Zookeeper session expired"); sessionOpen = false; listener.handleZooKeeperSessionDown(); break; case Disconnected: - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Lost connection to zookeeper server"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Lost connection to zookeeper server"); sessionOpen = false; listener.handleZooKeeperSessionDown(); break; case SyncConnected: - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Connection to zookeeper server established. Refetching master data"); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Connection to zookeeper server established. Refetching master data"); if (masterDataGatherer != null) { masterDataGatherer.restart(); } } switch (watchedEvent.getType()) { case NodeChildrenChanged: // Fleetcontrollers have either connected or disconnected to ZooKeeper - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeChildrenChanged"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeChildrenChanged"); break; case NodeDataChanged: // A fleetcontroller have changed what node it is voting for - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeDataChanged"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeDataChanged"); break; case NodeCreated: // How can this happen? Can one leave watches on non-existing nodes? - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeCreated"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeCreated"); break; case NodeDeleted: // We're not watching any nodes for whether they are deleted or not. - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeDeleted"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got unexpected ZooKeeper event NodeDeleted"); break; case None: if (state != null && state.equals(watchedEvent.getState())) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got None type event that didn't even alter session state. What does that indicate?"); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got None type event that didn't even alter session state. What does that indicate?"); } } state = watchedEvent.getState(); @@ -103,7 +103,7 @@ public class ZooKeeperDatabase extends Database { try{ this.listener = zksl; setupRoot(); - log.log(LogLevel.SPAM, "Fleetcontroller " + nodeIndex + ": Asking for initial data on master election"); + log.log(Level.FINEST, "Fleetcontroller " + nodeIndex + ": Asking for initial data on master election"); masterDataGatherer = new MasterDataGatherer(session, zooKeeperRoot, listener, nodeIndex); completedOk = true; } finally { @@ -114,13 +114,13 @@ public class ZooKeeperDatabase extends Database { private void createNode(String prefix, String nodename, byte value[]) throws KeeperException, InterruptedException { try{ if (session.exists(prefix + nodename, false) != null) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Zookeeper node '" + prefix + nodename + "' already exists. Not creating it"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Zookeeper node '" + prefix + nodename + "' already exists. Not creating it"); return; } session.create(prefix + nodename, value, acl, CreateMode.PERSISTENT); - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Created zookeeper node '" + prefix + nodename + "'"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Created zookeeper node '" + prefix + nodename + "'"); } catch (KeeperException.NodeExistsException e) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Node to create existed, " + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Node to create existed, " + "but this is normal as other nodes may create them at the same time."); } } @@ -139,14 +139,14 @@ public class ZooKeeperDatabase extends Database { createNode(zooKeeperRoot, "published_state_bundle", new byte[0]); // TODO dedupe string constants byte val[] = String.valueOf(nodeIndex).getBytes(utf8); deleteNodeIfExists(getMyIndexPath()); - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Creating ephemeral master vote node with vote to self."); session.create(getMyIndexPath(), val, acl, CreateMode.EPHEMERAL); } private void deleteNodeIfExists(String path) throws KeeperException, InterruptedException { if (session.exists(path, false) != null) { - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Removing master vote node."); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Removing master vote node."); session.delete(path, -1); } } @@ -162,11 +162,11 @@ public class ZooKeeperDatabase extends Database { public void close() { sessionOpen = false; try{ - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Trying to close ZooKeeper session 0x" + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Trying to close ZooKeeper session 0x" + Long.toHexString(session.getSessionId())); session.close(); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Got interrupt exception while closing session: " + e); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Got interrupt exception while closing session: " + e); } } @@ -178,7 +178,7 @@ public class ZooKeeperDatabase extends Database { if (sessionOpen && reportErrors) { StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); - log.log(LogLevel.WARNING, String.format("Fleetcontroller %s: %s. Exception: %s\n%s", + log.log(Level.WARNING, String.format("Fleetcontroller %s: %s. Exception: %s\n%s", nodeIndex, message, e.getMessage(), sw.toString())); } } @@ -187,7 +187,7 @@ public class ZooKeeperDatabase extends Database { byte val[] = String.valueOf(wantedMasterIndex).getBytes(utf8); try{ session.setData(getMyIndexPath(), val, -1); - log.log(LogLevel.INFO, "Fleetcontroller " + nodeIndex + ": Stored new vote in ephemeral node. " + nodeIndex + " -> " + wantedMasterIndex); + log.log(Level.INFO, "Fleetcontroller " + nodeIndex + ": Stored new vote in ephemeral node. " + nodeIndex + " -> " + wantedMasterIndex); return true; } catch (InterruptedException e) { throw (InterruptedException) new InterruptedException("Interrupted").initCause(e); @@ -199,7 +199,7 @@ public class ZooKeeperDatabase extends Database { public boolean storeLatestSystemStateVersion(int version) throws InterruptedException { byte data[] = Integer.toString(version).getBytes(utf8); try{ - log.log(LogLevel.INFO, String.format("Fleetcontroller %d: Storing new cluster state version in ZooKeeper: %d", nodeIndex, version)); + log.log(Level.INFO, String.format("Fleetcontroller %d: Storing new cluster state version in ZooKeeper: %d", nodeIndex, version)); var stat = session.setData(zooKeeperRoot + "latestversion", data, lastKnownStateVersionZNodeVersion); lastKnownStateVersionZNodeVersion = stat.getVersion(); return true; @@ -217,12 +217,12 @@ public class ZooKeeperDatabase extends Database { public Integer retrieveLatestSystemStateVersion() throws InterruptedException { Stat stat = new Stat(); try{ - log.log(LogLevel.DEBUG, () -> String.format("Fleetcontroller %d: Fetching latest cluster state at '%slatestversion'", + log.log(Level.FINE, () -> String.format("Fleetcontroller %d: Fetching latest cluster state at '%slatestversion'", nodeIndex, zooKeeperRoot)); byte[] data = session.getData(zooKeeperRoot + "latestversion", false, stat); lastKnownStateVersionZNodeVersion = stat.getVersion(); final Integer versionNumber = Integer.valueOf(new String(data, utf8)); - log.log(LogLevel.INFO, String.format("Fleetcontroller %d: Read cluster state version %d from ZooKeeper " + + log.log(Level.INFO, String.format("Fleetcontroller %d: Read cluster state version %d from ZooKeeper " + "(znode version %d)", nodeIndex, versionNumber, stat.getVersion())); return versionNumber; } catch (InterruptedException e) { @@ -249,7 +249,7 @@ public class ZooKeeperDatabase extends Database { } byte val[] = sb.toString().getBytes(utf8); try{ - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Storing wanted states at '" + zooKeeperRoot + "wantedstates'"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Storing wanted states at '" + zooKeeperRoot + "wantedstates'"); session.setData(zooKeeperRoot + "wantedstates", val, -1); return true; } catch (InterruptedException e) { @@ -262,7 +262,7 @@ public class ZooKeeperDatabase extends Database { public Map<Node, NodeState> retrieveWantedStates() throws InterruptedException { try{ - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Fetching wanted states at '" + zooKeeperRoot + "wantedstates'"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Fetching wanted states at '" + zooKeeperRoot + "wantedstates'"); Stat stat = new Stat(); byte[] data = session.getData(zooKeeperRoot + "wantedstates", false, stat); Map<Node, NodeState> wanted = new TreeMap<>(); @@ -277,7 +277,7 @@ public class ZooKeeperDatabase extends Database { NodeState nodeState = NodeState.deserialize(node.getType(), token.substring(colon + 1)); wanted.put(node, nodeState); } catch (Exception e) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Ignoring invalid wantedstate line in zookeeper '" + token + "'."); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Ignoring invalid wantedstate line in zookeeper '" + token + "'."); } } } @@ -300,7 +300,7 @@ public class ZooKeeperDatabase extends Database { } byte val[] = sb.toString().getBytes(utf8); try{ - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Storing start timestamps at '" + zooKeeperRoot + "starttimestamps"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Storing start timestamps at '" + zooKeeperRoot + "starttimestamps"); session.setData(zooKeeperRoot + "starttimestamps", val, -1); return true; } catch (InterruptedException e) { @@ -314,7 +314,7 @@ public class ZooKeeperDatabase extends Database { @Override public Map<Node, Long> retrieveStartTimestamps() throws InterruptedException { try{ - log.log(LogLevel.DEBUG, "Fleetcontroller " + nodeIndex + ": Fetching start timestamps at '" + zooKeeperRoot + "starttimestamps'"); + log.log(Level.FINE, "Fleetcontroller " + nodeIndex + ": Fetching start timestamps at '" + zooKeeperRoot + "starttimestamps'"); Stat stat = new Stat(); byte[] data = session.getData(zooKeeperRoot + "starttimestamps", false, stat); Map<Node, Long> wanted = new TreeMap<Node, Long>(); @@ -329,7 +329,7 @@ public class ZooKeeperDatabase extends Database { Long timestamp = Long.valueOf(token.substring(colon + 1)); wanted.put(n, timestamp); } catch (Exception e) { - log.log(LogLevel.WARNING, "Fleetcontroller " + nodeIndex + ": Ignoring invalid starttimestamp line in zookeeper '" + token + "'."); + log.log(Level.WARNING, "Fleetcontroller " + nodeIndex + ": Ignoring invalid starttimestamp line in zookeeper '" + token + "'."); } } } @@ -347,7 +347,7 @@ public class ZooKeeperDatabase extends Database { EnvelopedClusterStateBundleCodec envelopedBundleCodec = new SlimeClusterStateBundleCodec(); byte[] encodedBundle = envelopedBundleCodec.encodeWithEnvelope(stateBundle); try{ - log.log(LogLevel.DEBUG, () -> String.format("Fleetcontroller %d: Storing published state bundle %s at " + + log.log(Level.FINE, () -> String.format("Fleetcontroller %d: Storing published state bundle %s at " + "'%spublished_state_bundle' with expected znode version %d", nodeIndex, stateBundle, zooKeeperRoot, lastKnownStateBundleZNodeVersion)); var stat = session.setData(zooKeeperRoot + "published_state_bundle", encodedBundle, lastKnownStateBundleZNodeVersion); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java index eecdcc75228..94b1a9e1fbc 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.restapiv2.requests; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.time.TimeBudget; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; @@ -117,7 +117,7 @@ public class SetNodeStateRequest extends Request<SetResponse> { NodeStateChangeChecker.Result result = cluster.calculateEffectOfNewState( node, currentClusterState, condition, wantedState, newWantedState); - log.log(LogLevel.DEBUG, "node=" + node + + log.log(Level.FINE, "node=" + node + " current-cluster-state=" + currentClusterState + // Includes version in output format " condition=" + condition + " wanted-state=" + wantedState + diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java index c3c5c9e3b98..d8a72a9272c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RPCCommunicator.java @@ -14,7 +14,7 @@ import com.yahoo.jrt.Values; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.State; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.core.ActivateClusterStateVersionRequest; import com.yahoo.vespa.clustercontroller.core.ClusterStateBundle; import com.yahoo.vespa.clustercontroller.core.Communicator; @@ -110,7 +110,7 @@ public class RPCCommunicator implements Communicator { public void getNodeState(NodeInfo node, Waiter<GetNodeStateRequest> externalWaiter) { Target connection = getConnection(node); if ( ! connection.isValid()) { - log.log(LogLevel.DEBUG, () -> String.format("Connection to '%s' could not be created.", node.getRpcAddress())); + log.log(Level.FINE, () -> String.format("Connection to '%s' could not be created.", node.getRpcAddress())); } NodeState currentState = node.getReportedState(); Request req = new Request("getnodestate3"); @@ -138,7 +138,7 @@ public class RPCCommunicator implements Communicator { Target connection = getConnection(node); if ( ! connection.isValid()) { - log.log(LogLevel.DEBUG, () -> String.format("Connection to '%s' could not be created.", node.getRpcAddress())); + log.log(Level.FINE, () -> String.format("Connection to '%s' could not be created.", node.getRpcAddress())); return; } int nodeVersion = node.getVersion(); @@ -156,7 +156,7 @@ public class RPCCommunicator implements Communicator { v.add(new DataValue(encodedBundle.getCompression().data())); } - log.log(LogLevel.DEBUG, () -> String.format("Sending '%s' RPC to %s for state version %d", + log.log(Level.FINE, () -> String.format("Sending '%s' RPC to %s for state version %d", req.methodName(), node.getRpcAddress(), stateBundle.getVersion())); RPCSetClusterStateRequest stateRequest = new RPCSetClusterStateRequest(node, req, baselineState.getVersion()); waiter.setRequest(stateRequest); @@ -171,14 +171,14 @@ public class RPCCommunicator implements Communicator { Target connection = getConnection(node); if ( ! connection.isValid()) { - log.log(LogLevel.DEBUG, () -> String.format("Connection to '%s' could not be created.", node.getRpcAddress())); + log.log(Level.FINE, () -> String.format("Connection to '%s' could not be created.", node.getRpcAddress())); return; } var req = new Request(ACTIVATE_CLUSTER_STATE_VERSION_RPC_METHOD_NAME); req.parameters().add(new Int32Value(clusterStateVersion)); - log.log(LogLevel.DEBUG, () -> String.format("Sending '%s' RPC to %s for state version %d", + log.log(Level.FINE, () -> String.format("Sending '%s' RPC to %s for state version %d", req.methodName(), node.getRpcAddress(), clusterStateVersion)); var activationRequest = new RPCActivateClusterStateVersionRequest(node, req, clusterStateVersion); waiter.setRequest(activationRequest); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java index 87901a9bdb4..bdce2b31036 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/RpcServer.java @@ -15,7 +15,7 @@ import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.BackOffPolicy; import com.yahoo.jrt.slobrok.api.Register; import com.yahoo.jrt.slobrok.api.SlobrokList; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.net.HostName; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; @@ -97,13 +97,13 @@ public class RpcServer { public void connect() throws ListenFailedException, UnknownHostException { disconnect(); - log.log(LogLevel.DEBUG, "Fleetcontroller " + fleetControllerIndex + ": Connecting RPC server."); + log.log(Level.FINE, "Fleetcontroller " + fleetControllerIndex + ": Connecting RPC server."); if (supervisor != null) disconnect(); supervisor = new Supervisor(new Transport()); addMethods(); - log.log(LogLevel.DEBUG, "Fleetcontroller " + fleetControllerIndex + ": Attempting to bind to port " + port); + log.log(Level.FINE, "Fleetcontroller " + fleetControllerIndex + ": Attempting to bind to port " + port); acceptor = supervisor.listen(new Spec(port)); - log.log(LogLevel.DEBUG, "Fleetcontroller " + fleetControllerIndex + ": RPC server listening to port " + acceptor.port()); + log.log(Level.FINE, "Fleetcontroller " + fleetControllerIndex + ": RPC server listening to port " + acceptor.port()); StringBuffer slobroks = new StringBuffer("("); for (String s : slobrokConnectionSpecs) { slobroks.append(" ").append(s); @@ -112,7 +112,7 @@ public class RpcServer { SlobrokList slist = new SlobrokList(); slist.setup(slobrokConnectionSpecs); Spec spec = new Spec(HostName.getLocalhost(), acceptor.port()); - log.log(LogLevel.INFO, "Registering " + spec + " with slobrok at " + slobroks); + log.log(Level.INFO, "Registering " + spec + " with slobrok at " + slobroks); if (slobrokBackOffPolicy != null) { register = new Register(supervisor, slist, spec, slobrokBackOffPolicy); } else { @@ -123,7 +123,7 @@ public class RpcServer { public void disconnect() { if (register != null) { - log.log(LogLevel.DEBUG, "Fleetcontroller " + fleetControllerIndex + ": Disconnecting RPC server."); + log.log(Level.FINE, "Fleetcontroller " + fleetControllerIndex + ": Disconnecting RPC server."); register.shutdown(); register = null; } @@ -197,13 +197,13 @@ public class RpcServer { if (!e.getMessage().equals(lastConnectError) || time - lastConnectErrorTime > 60 * 1000) { lastConnectError = e.getMessage(); lastConnectErrorTime = time; - log.log(LogLevel.WARNING, "Failed to bind RPC server to port " + port +": " + e.getMessage()); + log.log(Level.WARNING, "Failed to bind RPC server to port " + port +": " + e.getMessage()); } } catch (Exception e) { if (!e.getMessage().equals(lastConnectError) || time - lastConnectErrorTime > 60 * 1000) { lastConnectError = e.getMessage(); lastConnectErrorTime = time; - log.log(LogLevel.WARNING, "Failed to initailize RPC server socket: " + e.getMessage()); + log.log(Level.WARNING, "Failed to initailize RPC server socket: " + e.getMessage()); } } } @@ -218,7 +218,7 @@ public class RpcServer { } try{ if (req.methodName().equals("getMaster")) { - log.log(LogLevel.DEBUG, "Resolving RPC getMaster request"); + log.log(Level.FINE, "Resolving RPC getMaster request"); Integer master = masterHandler.getMaster(); String masterReason = masterHandler.getMasterReason(); req.returnValues().add(new Int32Value(master == null ? -1 : master)); @@ -230,7 +230,7 @@ public class RpcServer { throw new IllegalStateException("Refusing to answer RPC calls as we are not the master fleetcontroller."); } if (req.methodName().equals("getNodeList")) { - log.log(LogLevel.DEBUG, "Resolving RPC getNodeList request"); + log.log(Level.FINE, "Resolving RPC getNodeList request"); List<String> slobrok = new ArrayList<String>(); List<String> rpc = new ArrayList<String>(); for(NodeInfo node : cluster.getNodeInfo()) { @@ -244,12 +244,12 @@ public class RpcServer { req.returnValues().add(new StringArray(rpc.toArray(new String[rpc.size()]))); req.returnRequest(); } else if (req.methodName().equals("getSystemState")) { - log.log(LogLevel.DEBUG, "Resolving RPC getSystemState request"); + log.log(Level.FINE, "Resolving RPC getSystemState request"); req.returnValues().add(new StringValue("")); req.returnValues().add(new StringValue(systemState.toString(true))); req.returnRequest(); } else if (req.methodName().equals("getNodeState")) { - log.log(LogLevel.DEBUG, "Resolving RPC getNodeState request"); + log.log(Level.FINE, "Resolving RPC getNodeState request"); NodeType nodeType = NodeType.get(req.parameters().get(0).asString()); int nodeIndex = req.parameters().get(1).asInt32(); @@ -296,20 +296,20 @@ public class RpcServer { changeListener.handleNewWantedNodeState(node, nodeState); } else { message = "Node " + node + " already had wanted state " + nodeState.toString(); - log.log(LogLevel.DEBUG, message); + log.log(Level.FINE, message); } req.returnValues().add(new StringValue(message)); req.returnRequest(); if (nodeState.getState() == State.UP && node.getPrematureCrashCount() > 0) { - log.log(LogLevel.INFO, "Clearing premature crash count of " + node.getPrematureCrashCount() + " as wanted state was set to up"); + log.log(Level.INFO, "Clearing premature crash count of " + node.getPrematureCrashCount() + " as wanted state was set to up"); node.setPrematureCrashCount(0); } } } catch (Exception e) { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); - log.log(LogLevel.DEBUG, "Failed RPC Request: " + sw); + log.log(Level.FINE, "Failed RPC Request: " + sw); } String errorMsg = e.getMessage(); if (errorMsg == null) { errorMsg = e.toString(); } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java index 825e6b033cd..c31fee36521 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/rpc/SlobrokClient.java @@ -8,7 +8,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.Node; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.core.NodeInfo; import com.yahoo.vespa.clustercontroller.core.NodeLookup; import com.yahoo.vespa.clustercontroller.core.Timer; @@ -73,8 +73,8 @@ public class SlobrokClient implements NodeLookup { if (freshMirror) { freshMirror = false; } else if (cluster.getSlobrokGenerationCount() == mirrorVersion) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Slobrok still at generation count " + cluster.getSlobrokGenerationCount() + ". Not updating."); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Slobrok still at generation count " + cluster.getSlobrokGenerationCount() + ". Not updating."); } return false; } @@ -142,7 +142,7 @@ public class SlobrokClient implements NodeLookup { cluster.setSlobrokGenerationCount(mirrorVersion); for (NodeInfo nodeInfo : cluster.getNodeInfo()) { if (slobrokNodes.containsKey(nodeInfo.getNode()) && nodeInfo.isRpcAddressOutdated()) { - log.log(LogLevel.WARNING, "Node " + nodeInfo + log.log(Level.WARNING, "Node " + nodeInfo + " was tagged NOT in slobrok even though it is. It was in the following lists:" + (newNodes.contains(nodeInfo.getNode()) ? " newNodes" : "") + (missingNodeInfos.contains(nodeInfo) ? " missingNodes" : "") @@ -151,7 +151,7 @@ public class SlobrokClient implements NodeLookup { nodeInfo.markRpcAddressLive(); } } - log.log(LogLevel.SPAM, "Slobrok information updated to generation " + cluster.getSlobrokGenerationCount()); + log.log(Level.FINEST, "Slobrok information updated to generation " + cluster.getSlobrokGenerationCount()); return true; } @@ -196,7 +196,7 @@ public class SlobrokClient implements NodeLookup { private Map<Node, SlobrokData> getSlobrokData(String pattern) { Map<Node, SlobrokData> result = new TreeMap<>(); List<Mirror.Entry> entries = mirror.lookup(pattern); - log.log(LogLevel.SPAM, "Looking for slobrok entries with pattern '" + pattern + "'. Found " + entries.size() + " entries."); + log.log(Level.FINEST, "Looking for slobrok entries with pattern '" + pattern + "'. Found " + entries.size() + " entries."); for (Mirror.Entry entry : entries) { StringTokenizer st = new StringTokenizer(entry.getName(), "/"); String addressType = st.nextToken(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java index 6e4c0bc7eaa..3d3de32c356 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/status/statuspage/StatusPageServer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.clustercontroller.core.status.statuspage; import com.yahoo.exception.ExceptionUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.BufferedReader; import java.io.IOException; @@ -58,7 +58,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { if (ssocket != null && ssocket.isBound() && (ssocket.getLocalPort() == port || port == 0)) { return true; } else { - log.log(LogLevel.SPAM, "Status page server socket is no longer connected: "+ (ssocket != null) + " " + ssocket.isBound() + " " + ssocket.getLocalPort() + " " + port); + log.log(Level.FINEST, "Status page server socket is no longer connected: "+ (ssocket != null) + " " + ssocket.isBound() + " " + ssocket.getLocalPort() + " " + port); return false; } } @@ -83,9 +83,9 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { Thread.sleep(10); } if (!isConnected()) { - log.log(LogLevel.INFO, "Fleetcontroller: Server Socket not ready after connect()"); + log.log(Level.INFO, "Fleetcontroller: Server Socket not ready after connect()"); } - log.log(LogLevel.DEBUG, "Fleet controller status page viewer listening to " + ssocket.getLocalSocketAddress()); + log.log(Level.FINE, "Fleet controller status page viewer listening to " + ssocket.getLocalSocketAddress()); monitor.notifyAll(); } } @@ -102,7 +102,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { public void setPort(int port) throws java.io.IOException, InterruptedException { // Only bother to reconnect if we were connected to begin with, we care about what port it runs on, and it's not already running there if (port != 0 && isConnected() && port != ((InetSocketAddress) ssocket.getLocalSocketAddress()).getPort()) { - log.log(LogLevel.INFO, "Exchanging port used by status server. Moving from port " + log.log(Level.INFO, "Exchanging port used by status server. Moving from port " + ((InetSocketAddress) ssocket.getLocalSocketAddress()).getPort() + " to port " + port); disconnect(); this.port = port; @@ -146,10 +146,10 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { } catch (SocketTimeoutException e) { // Ignore, since timeout is set to 100 ms } catch (java.io.IOException e) { - log.log(shouldBeConnected ? LogLevel.WARNING : LogLevel.DEBUG, "Caught IO exception in ServerSocket.accept(): " + e.getMessage()); + log.log(shouldBeConnected ? Level.WARNING : Level.FINE, "Caught IO exception in ServerSocket.accept(): " + e.getMessage()); } if (connection == null) continue; - log.log(LogLevel.DEBUG, "Got a status page request."); + log.log(Level.FINE, "Got a status page request."); String requestString = ""; OutputStream output = null; try (BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream()))) { @@ -168,7 +168,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { if (s == null || s.equals("")) break; sb.append(s).append("\n"); } - log.log(LogLevel.DEBUG, "Got HTTP request: " + sb.toString()); + log.log(Level.FINE, "Got HTTP request: " + sb.toString()); HttpRequest httpRequest = null; StatusPageResponse response = null; @@ -235,27 +235,27 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { output.write(header.toString().getBytes()); output.write(response.getOutputStream().toByteArray()); } catch (java.io.IOException e) { - log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? LogLevel.DEBUG : LogLevel.INFO, + log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? Level.FINE : Level.INFO, "Failed to process HTTP request : " + e.getMessage()); } catch (Exception e) { - log.log(LogLevel.WARNING, "Caught exception in HTTP server thread: " + log.log(Level.WARNING, "Caught exception in HTTP server thread: " + e.getClass().getName() + ": " + e.getMessage()); } finally { if (output != null) try { output.close(); } catch (IOException e) { - log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? LogLevel.DEBUG : LogLevel.INFO, + log.log(e.getMessage().indexOf("Broken pipe") >= 0 ? Level.FINE : Level.INFO, "Failed to close output stream on socket " + connection + ": " + e.getMessage()); } if (connection != null) try{ connection.close(); } catch (IOException e) { - log.log(LogLevel.INFO, "Failed to close socket " + connection + ": " + e.getMessage()); + log.log(Level.INFO, "Failed to close socket " + connection + ": " + e.getMessage()); } } } } catch (InterruptedException e) { - log.log(LogLevel.DEBUG, "Status processing thread shut down by interrupt exception: " + e); + log.log(Level.FINE, "Status processing thread shut down by interrupt exception: " + e); } } @@ -392,7 +392,7 @@ public class StatusPageServer implements Runnable, StatusPageServerInterface { if (!e.getMessage().equals(lastConnectError) || time - lastConnectErrorTime > 60 * 1000) { lastConnectError = e.getMessage(); lastConnectErrorTime = time; - log.log(LogLevel.WARNING, "Failed to initialize HTTP status server server socket: " + e.getMessage()); + log.log(Level.WARNING, "Failed to initialize HTTP status server server socket: " + e.getMessage()); } } } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java index b611d573ff5..0cf84cfe54c 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/DummyVdsNode.java @@ -15,7 +15,7 @@ import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.BackOffPolicy; import com.yahoo.jrt.slobrok.api.Register; import com.yahoo.jrt.slobrok.api.SlobrokList; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.*; import com.yahoo.vespa.clustercontroller.core.rpc.RPCCommunicator; import com.yahoo.vespa.clustercontroller.core.rpc.RPCUtil; @@ -84,7 +84,7 @@ public class DummyVdsNode { private Thread messageResponder = new Thread() { public void run() { - log.log(LogLevel.DEBUG, "Dummy node " + DummyVdsNode.this.toString() + ": starting message reponder thread"); + log.log(Level.FINE, "Dummy node " + DummyVdsNode.this.toString() + ": starting message reponder thread"); while (true) { synchronized (timer) { if (isInterrupted()) break; @@ -92,7 +92,7 @@ public class DummyVdsNode { for (Iterator<Req> it = waitingRequests.iterator(); it.hasNext(); ) { Req r = it.next(); if (r.timeout <= currentTime) { - log.log(LogLevel.DEBUG, "Dummy node " + DummyVdsNode.this.toString() + ": Responding to node state request at time " + currentTime); + log.log(Level.FINE, "Dummy node " + DummyVdsNode.this.toString() + ": Responding to node state request at time " + currentTime); r.request.returnValues().add(new StringValue(nodeState.serialize())); if (r.request.methodName().equals("getnodestate3")) { r.request.returnValues().add(new StringValue("No host info in dummy implementation")); @@ -109,7 +109,7 @@ public class DummyVdsNode { } } } - log.log(LogLevel.DEBUG, "Dummy node " + DummyVdsNode.this.toString() + ": shut down message reponder thread"); + log.log(Level.FINE, "Dummy node " + DummyVdsNode.this.toString() + ": shut down message reponder thread"); } }; @@ -171,7 +171,7 @@ public class DummyVdsNode { void disconnectBreakConnection() { disconnect(true, FleetControllerTest.timeoutMS, false); } void disconnectAsShutdown() { disconnect(true, FleetControllerTest.timeoutMS, true); } private void disconnect(boolean waitForPendingNodeStateRequest, long timeoutms, boolean setStoppingStateFirst) { - log.log(LogLevel.DEBUG, "Dummy node " + DummyVdsNode.this.toString() + ": Breaking connection." + (waitForPendingNodeStateRequest ? " Waiting for pending state first." : "")); + log.log(Level.FINE, "Dummy node " + DummyVdsNode.this.toString() + ": Breaking connection." + (waitForPendingNodeStateRequest ? " Waiting for pending state first." : "")); if (waitForPendingNodeStateRequest) { this.waitForPendingGetNodeStateRequest(timeoutms); } @@ -190,7 +190,7 @@ public class DummyVdsNode { acceptor.shutdown().join(); supervisor.transport().shutdown().join(); supervisor = null; - log.log(LogLevel.DEBUG, "Dummy node " + DummyVdsNode.this.toString() + ": Done breaking connection."); + log.log(Level.FINE, "Dummy node " + DummyVdsNode.this.toString() + ": Done breaking connection."); } public String toString() { @@ -244,25 +244,25 @@ public class DummyVdsNode { private void waitForPendingGetNodeStateRequest(long timeout) { long startTime = System.currentTimeMillis(); long endTime = startTime + timeout; - log.log(LogLevel.DEBUG, "Dummy node " + this + " waiting for pending node state request."); + log.log(Level.FINE, "Dummy node " + this + " waiting for pending node state request."); while (true) { synchronized(timer) { if (!waitingRequests.isEmpty()) { - log.log(LogLevel.DEBUG, "Dummy node " + this + " has pending request, returning."); + log.log(Level.FINE, "Dummy node " + this + " has pending request, returning."); return; } try{ - log.log(LogLevel.DEBUG, "Dummy node " + this + " waiting " + (endTime - startTime) + " ms for pending request."); + log.log(Level.FINE, "Dummy node " + this + " waiting " + (endTime - startTime) + " ms for pending request."); timer.wait(endTime - startTime); } catch (InterruptedException e) { } - log.log(LogLevel.DEBUG, "Dummy node " + this + " woke up to recheck."); + log.log(Level.FINE, "Dummy node " + this + " woke up to recheck."); } startTime = System.currentTimeMillis(); if (startTime >= endTime) { - log.log(LogLevel.DEBUG, "Dummy node " + this + " timeout passed. Don't have pending request."); + log.log(Level.FINE, "Dummy node " + this + " timeout passed. Don't have pending request."); if (!waitingRequests.isEmpty()) { - log.log(LogLevel.DEBUG, "Dummy node " + this + ". Non-empty set of waiting requests"); + log.log(Level.FINE, "Dummy node " + this + ". Non-empty set of waiting requests"); } throw new IllegalStateException("Timeout. No pending get node state request pending after waiting " + timeout + " milliseconds."); } @@ -271,7 +271,7 @@ public class DummyVdsNode { void replyToPendingNodeStateRequests() { for(Req req : waitingRequests) { - log.log(LogLevel.DEBUG, "Dummy node " + this + " answering pending node state request."); + log.log(Level.FINE, "Dummy node " + this + " answering pending node state request."); req.request.returnValues().add(new StringValue(nodeState.serialize())); if (req.request.methodName().equals("getnodestate3")) { req.request.returnValues().add(new StringValue(hostInfo)); @@ -283,7 +283,7 @@ public class DummyVdsNode { } public void setNodeState(NodeState state, String hostInfo) { - log.log(LogLevel.DEBUG, "Dummy node " + this + " got new state: " + state); + log.log(Level.FINE, "Dummy node " + this + " got new state: " + state); synchronized(timer) { this.nodeState = state; this.hostInfo = hostInfo; @@ -402,7 +402,7 @@ public class DummyVdsNode { private void rpc_storageConnect(Request req) { synchronized(timer) { - log.log(LogLevel.SPAM, "Dummy node " + this + " got old type handle connect message."); + log.log(Level.FINEST, "Dummy node " + this + " got old type handle connect message."); req.returnValues().add(new Int32Value(0)); negotiatedHandle = true; } @@ -415,7 +415,7 @@ public class DummyVdsNode { return; } String stateString = nodeState.serialize(-1, true); - log.log(LogLevel.DEBUG, "Dummy node " + this + " got old type get node state request, answering: " + stateString); + log.log(Level.FINE, "Dummy node " + this + " got old type get node state request, answering: " + stateString); req.returnValues().add(new Int32Value(1)); req.returnValues().add(new StringValue("")); req.returnValues().add(new StringValue(stateString)); @@ -427,7 +427,7 @@ public class DummyVdsNode { for (Iterator<Req> it = waitingRequests.iterator(); it.hasNext(); ) { Req r = it.next(); if (r.request.parameters().size() > 2 && r.request.parameters().get(2).asInt32() == index) { - log.log(LogLevel.DEBUG, "Dummy node " + DummyVdsNode.this.toString() + ": Responding to node state reply from controller " + index + " as we received new one"); + log.log(Level.FINE, "Dummy node " + DummyVdsNode.this.toString() + ": Responding to node state reply from controller " + index + " as we received new one"); r.request.returnValues().add(new StringValue(nodeState.serialize())); r.request.returnValues().add(new StringValue("No host info from dummy implementation")); r.request.returnRequest(); @@ -440,7 +440,7 @@ public class DummyVdsNode { } private void rpc_getNodeState2(Request req) { - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Got " + req.methodName() + " request"); + log.log(Level.FINE, "Dummy node " + this + ": Got " + req.methodName() + " request"); try{ String oldState = req.parameters().get(0).asString(); int timeout = req.parameters().get(1).asInt32(); @@ -452,14 +452,14 @@ public class DummyVdsNode { boolean sentReply = sendGetNodeStateReply(index); NodeState givenState = (oldState.equals("unknown") ? null : NodeState.deserialize(type, oldState)); if (givenState != null && (givenState.equals(nodeState) || sentReply)) { - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Has same state as reported " + givenState + ". Queing request. Timeout is " + timeout + " ms. " + log.log(Level.FINE, "Dummy node " + this + ": Has same state as reported " + givenState + ". Queing request. Timeout is " + timeout + " ms. " + "Will be answered at time " + (timer.getCurrentTimeInMillis() + timeout * 800l / 1000)); req.detach(); waitingRequests.add(new Req(req, timer.getCurrentTimeInMillis() + timeout * 800l / 1000)); - log.log(LogLevel.DEBUG, "Dummy node " + this + " has now " + waitingRequests.size() + " entries and is " + (waitingRequests.isEmpty() ? "empty" : "not empty")); + log.log(Level.FINE, "Dummy node " + this + " has now " + waitingRequests.size() + " entries and is " + (waitingRequests.isEmpty() ? "empty" : "not empty")); timer.notifyAll(); } else { - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Request had " + (givenState == null ? "no state" : "different state(" + givenState +")") + ". Answering with " + nodeState); + log.log(Level.FINE, "Dummy node " + this + ": Request had " + (givenState == null ? "no state" : "different state(" + givenState +")") + ". Answering with " + nodeState); req.returnValues().add(new StringValue(nodeState.serialize())); if (req.methodName().equals("getnodestate3")) { req.returnValues().add(new StringValue("Dummy node host info")); @@ -468,7 +468,7 @@ public class DummyVdsNode { } } } catch (Exception e) { - log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering " + req.methodName() + " request: " + e.getMessage()); + log.log(Level.SEVERE, "Dummy node " + this + ": An error occurred when answering " + req.methodName() + " request: " + e.getMessage()); e.printStackTrace(System.err); req.setError(ErrorCode.METHOD_FAILED, e.getMessage()); } @@ -526,9 +526,9 @@ public class DummyVdsNode { } req.returnValues().add(new Int32Value(1)); req.returnValues().add(new StringValue("OK")); - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Got new system state (through old setsystemstate call) " + newState); + log.log(Level.FINE, "Dummy node " + this + ": Got new system state (through old setsystemstate call) " + newState); } catch (Exception e) { - log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering setsystemstate request: " + e.getMessage()); + log.log(Level.SEVERE, "Dummy node " + this + ": An error occurred when answering setsystemstate request: " + e.getMessage()); e.printStackTrace(System.err); req.returnValues().add(new Int32Value(ErrorCode.METHOD_FAILED)); req.returnValues().add(new StringValue(e.getMessage())); @@ -550,9 +550,9 @@ public class DummyVdsNode { } timer.notifyAll(); } - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Got new system state " + newState); + log.log(Level.FINE, "Dummy node " + this + ": Got new system state " + newState); } catch (Exception e) { - log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering setsystemstate request: " + e.getMessage()); + log.log(Level.SEVERE, "Dummy node " + this + ": An error occurred when answering setsystemstate request: " + e.getMessage()); e.printStackTrace(System.err); req.setError(ErrorCode.METHOD_FAILED, e.getMessage()); } @@ -573,9 +573,9 @@ public class DummyVdsNode { } timer.notifyAll(); } - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Got new cluster state " + stateBundle); + log.log(Level.FINE, "Dummy node " + this + ": Got new cluster state " + stateBundle); } catch (Exception e) { - log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering setdistributionstates request: " + e.getMessage()); + log.log(Level.SEVERE, "Dummy node " + this + ": An error occurred when answering setdistributionstates request: " + e.getMessage()); e.printStackTrace(System.err); req.setError(ErrorCode.METHOD_FAILED, e.getMessage()); } @@ -596,13 +596,13 @@ public class DummyVdsNode { activatedClusterStateVersion = activateVersion; timer.notifyAll(); } else { - log.log(LogLevel.DEBUG, () -> String.format("Dummy node %s: got a mismatching activation (request version %d, " + + log.log(Level.FINE, () -> String.format("Dummy node %s: got a mismatching activation (request version %d, " + "actual %d), not marking version as active", this, activateVersion, actualVersion)); } } - log.log(LogLevel.DEBUG, "Dummy node " + this + ": Activating cluster state version " + activateVersion); + log.log(Level.FINE, "Dummy node " + this + ": Activating cluster state version " + activateVersion); } catch (Exception e) { - log.log(LogLevel.ERROR, "Dummy node " + this + ": An error occurred when answering activate_cluster_state_version request: " + e.getMessage()); + log.log(Level.SEVERE, "Dummy node " + this + ": An error occurred when answering activate_cluster_state_version request: " + e.getMessage()); e.printStackTrace(System.err); req.setError(ErrorCode.METHOD_FAILED, e.getMessage()); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java index e161c4aacb7..bda24b826fd 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FakeTimer.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.clustercontroller.core.testutils.LogFormatter; import java.util.logging.Logger; @@ -23,7 +23,7 @@ public class FakeTimer implements Timer { public synchronized void advanceTime(long time) { long currentTime = getCurrentTimeInMillis(); this.currentTime += time; - log.log(LogLevel.DEBUG, "Time advanced by " + time + " ms. Time increased from " + currentTime + " to " + (currentTime + time)); + log.log(Level.FINE, "Time advanced by " + time + " ms. Time increased from " + currentTime + " to " + (currentTime + time)); notifyAll(); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java index 059d7d68754..5c910983bc9 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/FleetControllerTest.java @@ -9,7 +9,7 @@ import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.BackOffPolicy; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.state.ClusterState; @@ -135,13 +135,13 @@ public abstract class FleetControllerTest implements Waiter { } void setUpSystem(boolean useFakeTimer, FleetControllerOptions options) throws Exception { - log.log(LogLevel.DEBUG, "Setting up system"); + log.log(Level.FINE, "Setting up system"); slobrok = new Slobrok(); this.options = options; if (options.zooKeeperServerAddress != null) { zooKeeperServer = new ZooKeeperTestServer(); this.options.zooKeeperServerAddress = zooKeeperServer.getAddress(); - log.log(LogLevel.DEBUG, "Set up new zookeeper server at " + this.options.zooKeeperServerAddress); + log.log(Level.FINE, "Set up new zookeeper server at " + this.options.zooKeeperServerAddress); } this.options.slobrokConnectionSpecs = new String[1]; this.options.slobrokConnectionSpecs[0] = "tcp/localhost:" + slobrok.port(); @@ -211,7 +211,7 @@ public abstract class FleetControllerTest implements Waiter { if (fleetController == null) { fleetController = createFleetController(usingFakeTimer, options, true, null); } else { - log.log(LogLevel.WARNING, "already started fleetcontroller, not starting another"); + log.log(Level.WARNING, "already started fleetcontroller, not starting another"); } } @@ -299,7 +299,7 @@ public abstract class FleetControllerTest implements Waiter { protected void tearDownSystem() throws Exception { if (testName != null) { - //log.log(LogLevel.INFO, "STOPPING TEST " + testName); + //log.log(Level.INFO, "STOPPING TEST " + testName); System.err.println("STOPPING TEST " + testName); testName = null; } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java index f919bd867c6..557765ca761 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MasterElectionTest.java @@ -7,7 +7,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.NodeState; import com.yahoo.vdslib.state.NodeType; @@ -119,18 +119,18 @@ public class MasterElectionTest extends FleetControllerTest { @Ignore public void testMasterElection() throws Exception { startingTest("MasterElectionTest::testMasterElection"); - log.log(LogLevel.INFO, "STARTING TEST: MasterElectionTest::testMasterElection()"); + log.log(Level.INFO, "STARTING TEST: MasterElectionTest::testMasterElection()"); FleetControllerOptions options = defaultOptions("mycluster"); options.masterZooKeeperCooldownPeriod = 1; setUpFleetController(5, false, options); waitForMaster(0); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 0"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 0"); fleetControllers.get(0).shutdown(); waitForMaster(1); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 1"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 1"); fleetControllers.get(1).shutdown(); waitForMaster(2); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 2"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 2"); fleetControllers.get(2).shutdown(); // Too few for there to be a master at this point @@ -139,23 +139,23 @@ public class MasterElectionTest extends FleetControllerTest { assertFalse("Fleet controller " + i, fleetControllers.get(i).isMaster()); } - log.log(LogLevel.INFO, "STARTING FLEET CONTROLLER 2"); + log.log(Level.INFO, "STARTING FLEET CONTROLLER 2"); fleetControllers.set(2, createFleetController(usingFakeTimer, fleetControllers.get(2).getOptions(), true, null)); waitForMaster(2); - log.log(LogLevel.INFO, "STARTING FLEET CONTROLLER 0"); + log.log(Level.INFO, "STARTING FLEET CONTROLLER 0"); fleetControllers.set(0, createFleetController(usingFakeTimer, fleetControllers.get(0).getOptions(), true, null)); waitForMaster(0); - log.log(LogLevel.INFO, "STARTING FLEET CONTROLLER 1"); + log.log(Level.INFO, "STARTING FLEET CONTROLLER 1"); fleetControllers.set(1, createFleetController(usingFakeTimer, fleetControllers.get(1).getOptions(), true, null)); waitForMaster(0); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 4"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 4"); fleetControllers.get(4).shutdown(); waitForMaster(0); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 3"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 3"); fleetControllers.get(3).shutdown(); waitForMaster(0); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 2"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 2"); fleetControllers.get(2).shutdown(); // Too few for there to be a master at this point @@ -166,20 +166,20 @@ public class MasterElectionTest extends FleetControllerTest { } private void waitForMaster(int master) { - log.log(LogLevel.INFO, "Entering waitForMaster"); + log.log(Level.INFO, "Entering waitForMaster"); boolean isOnlyMaster = false; for (int i=0; i < FleetControllerTest.timeoutMS; i+=100) { if (!fleetControllers.get(master).isMaster()) { - log.log(LogLevel.INFO, "Node " + master + " is not master yet, sleeping more"); + log.log(Level.INFO, "Node " + master + " is not master yet, sleeping more"); timer.advanceTime(100); waitForCompleteCycle(master); } else { - log.log(LogLevel.INFO, "Node " + master + " is master. Checking that noone else is master"); + log.log(Level.INFO, "Node " + master + " is master. Checking that noone else is master"); isOnlyMaster = true; for (int j=0; j<fleetControllers.size(); ++j) { if (j != master && fleetControllers.get(j).isMaster()) { isOnlyMaster = false; - log.log(LogLevel.INFO, "Node " + j + " also says it is master."); + log.log(Level.INFO, "Node " + j + " also says it is master."); } } @@ -192,11 +192,11 @@ public class MasterElectionTest extends FleetControllerTest { } if (!isOnlyMaster) { - log.log(LogLevel.INFO, "Node " + master + " is not the only master"); + log.log(Level.INFO, "Node " + master + " is not the only master"); throw new IllegalStateException("Node " + master + " never got to be the only master."); } - log.log(LogLevel.INFO, "Leaving waitForMaster"); + log.log(Level.INFO, "Leaving waitForMaster"); } private static class StrictlyIncreasingVersionChecker { @@ -264,9 +264,9 @@ public class MasterElectionTest extends FleetControllerTest { zooKeeperServer = ZooKeeperTestServer.createWithFixedPort(18342); timer.advanceTime(10 * 1000); // Wait long enough for fleetcontroller wanting to retry zookeeper connection - log.log(LogLevel.INFO, "WAITING FOR 0 TO BE MASTER"); + log.log(Level.INFO, "WAITING FOR 0 TO BE MASTER"); waitForMaster(0); - log.log(LogLevel.INFO, "SHUTTING DOWN"); + log.log(Level.INFO, "SHUTTING DOWN"); } @Test @@ -278,28 +278,28 @@ public class MasterElectionTest extends FleetControllerTest { setUpFleetController(5, false, options); waitForMaster(0); - log.log(LogLevel.INFO, "STOPPING ZOOKEEPER SERVER AT " + zooKeeperServer.getAddress()); + log.log(Level.INFO, "STOPPING ZOOKEEPER SERVER AT " + zooKeeperServer.getAddress()); zooKeeperServer.shutdown(true); waitForCompleteCycles(); timer.advanceTime(options.zooKeeperSessionTimeout); waitForZookeeperDisconnected(); // Noone can be master if server is unavailable - log.log(LogLevel.INFO, "Checking master status"); + log.log(Level.INFO, "Checking master status"); for (int i=0; i<fleetControllers.size(); ++i) { assertFalse("Index " + i, fleetControllers.get(i).isMaster()); } zooKeeperServer = new ZooKeeperTestServer(); - log.log(LogLevel.INFO, "STARTED ZOOKEEPER SERVER AT " + zooKeeperServer.getAddress()); + log.log(Level.INFO, "STARTED ZOOKEEPER SERVER AT " + zooKeeperServer.getAddress()); for (FleetController fc : fleetControllers) { FleetControllerOptions myoptions = fc.getOptions(); myoptions.zooKeeperServerAddress = zooKeeperServer.getAddress(); fc.updateOptions(myoptions, 0); - log.log(LogLevel.INFO, "Should now have sent out new zookeeper server address " + myoptions.zooKeeperServerAddress + " to fleetcontroller " + myoptions.fleetControllerIndex); + log.log(Level.INFO, "Should now have sent out new zookeeper server address " + myoptions.zooKeeperServerAddress + " to fleetcontroller " + myoptions.fleetControllerIndex); } timer.advanceTime(10 * 1000); // Wait long enough for fleetcontroller wanting to retry zookeeper connection waitForMaster(0); - log.log(LogLevel.INFO, "SHUTTING DOWN"); + log.log(Level.INFO, "SHUTTING DOWN"); } /** Ignored for unknown reasons */ @@ -313,7 +313,7 @@ public class MasterElectionTest extends FleetControllerTest { waitForMaster(0); timer.advanceTime(24 * 3600 * 1000); // A day waitForCompleteCycle(1); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 0"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 0"); fleetControllers.get(0).shutdown(); waitForCompleteCycle(1); // 5 minutes is not long enough period to wait before letting this node be master. @@ -381,7 +381,7 @@ public class MasterElectionTest extends FleetControllerTest { assertEquals(req.toString(), "All 3 nodes agree that 0 is current master.", req.returnValues().get(1).asString()); } - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 0"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 0"); fleetControllers.get(0).shutdown(); // Wait until fc 1 & 2 votes for node 1 waitForCompleteCycle(1); @@ -439,7 +439,7 @@ public class MasterElectionTest extends FleetControllerTest { fleetControllers.get(i).updateOptions(nodeOptions, 2); } waitForMaster(0); - log.log(LogLevel.INFO, "SHUTTING DOWN FLEET CONTROLLER 0"); + log.log(Level.INFO, "SHUTTING DOWN FLEET CONTROLLER 0"); fleetControllers.get(0).shutdown(); waitForMaster(1); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java index 532919f05da..73879d3d159 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java @@ -10,7 +10,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.distribution.ConfiguredNode; import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.state.ClusterState; @@ -101,7 +101,7 @@ public class RpcServerTest extends FleetControllerTest { waitForStableSystem(); assertEquals(true, nodes.get(0).isDistributor()); - log.log(LogLevel.INFO, "Disconnecting distributor 0. Waiting for state to reflect change."); + log.log(Level.INFO, "Disconnecting distributor 0. Waiting for state to reflect change."); nodes.get(0).disconnect(); nodes.get(19).disconnect(); fleetController.waitForNodesInSlobrok(9, 9, timeoutMS); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java index 209da523705..5a96af90d68 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/SlobrokTest.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -45,19 +45,19 @@ public class SlobrokTest extends FleetControllerTest { // Test that we survive some slobrok instability without changing system state. for (int j=0; j<4; ++j) { - log.log(LogLevel.INFO, "Mirror updateForDistributor count is " + fleetController.getSlobrokMirrorUpdates()); - log.log(LogLevel.INFO, "STOPPING SLOBROK SERVER (" + (j+1) + "/4)"); + log.log(Level.INFO, "Mirror updateForDistributor count is " + fleetController.getSlobrokMirrorUpdates()); + log.log(Level.INFO, "STOPPING SLOBROK SERVER (" + (j+1) + "/4)"); slobrok.stop(); for (int i=0; i<10; ++i) { // Force one node to at least notice that the slobrok server is gone if (i == 5) { - log.log(LogLevel.INFO, "Forcing one node to initate a resend: " + nodes.get(3)); + log.log(Level.INFO, "Forcing one node to initate a resend: " + nodes.get(3)); nodes.get(3).replyToPendingNodeStateRequests(); } waitForCompleteCycle(); timer.advanceTime(100); } - log.log(LogLevel.INFO, "STARTING SLOBROK SERVER AGAIN (" + (j+1) + "/4)"); + log.log(Level.INFO, "STARTING SLOBROK SERVER AGAIN (" + (j+1) + "/4)"); slobrok = new Slobrok(slobrokPort); // May take up to 30 seconds for slobrok clients to re-register. Trigger retry. for (DummyVdsNode node : nodes) { @@ -68,7 +68,7 @@ public class SlobrokTest extends FleetControllerTest { waitForCompleteCycle(); fleetController.waitForNodesInSlobrok(10, 10, timeoutMS); - log.log(LogLevel.INFO, "Waiting for cluster to be up and available again"); + log.log(Level.INFO, "Waiting for cluster to be up and available again"); for (int i = 0; i < timeoutMS; i += 10) { if (clusterAvailable()) break; timer.advanceTime(1000); @@ -96,7 +96,7 @@ public class SlobrokTest extends FleetControllerTest { int version = fleetController.getSystemState().getVersion(); nodes.get(0).disconnectSlobrok(); - log.log(LogLevel.INFO, "DISCONNECTED NODE FROM SLOBROK. SHOULD BE IN COOLDOWN PERIOD"); + log.log(Level.INFO, "DISCONNECTED NODE FROM SLOBROK. SHOULD BE IN COOLDOWN PERIOD"); fleetController.waitForNodesInSlobrok(9, 10, timeoutMS); synchronized (timer) { nodes.get(0).sendGetNodeStateReply(0); @@ -106,7 +106,7 @@ public class SlobrokTest extends FleetControllerTest { timer.advanceTime(1000); try{ Thread.sleep(10); } catch (InterruptedException e) {} assertEquals(version, fleetController.getSystemState().getVersion()); - log.log(LogLevel.INFO, "JUMPING TIME. NODE SHOULD BE MARKED DOWN"); + log.log(Level.INFO, "JUMPING TIME. NODE SHOULD BE MARKED DOWN"); // At this point the fleetcontroller might not have noticed that the node is out of slobrok yet. // Thus we keep advancing time another minute such that it should get down. timer.advanceTime(options.nodeStateRequestTimeoutMS + options.maxSlobrokDisconnectGracePeriod); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java index 398c1c16473..7e2d8d09e1b 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/StateGatherTest.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.junit.Test; import java.util.concurrent.TimeoutException; @@ -25,7 +25,7 @@ public class StateGatherTest extends FleetControllerTest { @Test public void testAlwaysHavePendingGetNodeStateRequestTowardsNodes() throws Exception { - Logger.getLogger(NodeStateGatherer.class.getName()).setLevel(LogLevel.SPAM); + Logger.getLogger(NodeStateGatherer.class.getName()).setLevel(Level.FINEST); startingTest("StateGatherTest::testOverlappingGetNodeStateRequests"); FleetControllerOptions options = defaultOptions("mycluster"); options.nodeStateRequestTimeoutMS = 10 * 60 * 1000; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java index deef3157eed..b44edd0d209 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/testutils/Waiter.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.core.testutils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vdslib.state.ClusterState; import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.core.DummyVdsNode; @@ -88,7 +88,7 @@ public interface Waiter { } public final void wait(WaitCondition c, WaitTask wt, int timeoutMS) { - log.log(LogLevel.INFO, "Waiting for " + c + (wt == null ? "" : " with wait task " + wt)); + log.log(Level.INFO, "Waiting for " + c + (wt == null ? "" : " with wait task " + wt)); final long startTime = System.currentTimeMillis(); final long endTime = startTime + timeoutMS; String lastReason = null; @@ -96,11 +96,11 @@ public interface Waiter { synchronized (data.getMonitor()) { String reason = c.isConditionMet(); if (reason == null) { - log.log(LogLevel.INFO, "Condition met. Returning"); + log.log(Level.INFO, "Condition met. Returning"); return; } if (lastReason == null || !lastReason.equals(reason)) { - log.log(LogLevel.INFO, "Wait condition not met: " + reason); + log.log(Level.INFO, "Wait condition not met: " + reason); lastReason = reason; } try { diff --git a/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java b/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java index 2d6880e0551..3242041b287 100644 --- a/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java +++ b/clustercontroller-standalone/src/main/java/com/yahoo/vespa/clustercontroller/standalone/StandAloneClusterController.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.clustercontroller.standalone; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.log.event.Event; import com.yahoo.vespa.clustercontroller.core.FleetController; @@ -31,7 +31,7 @@ public class StandAloneClusterController { try{ app.stop(); } catch (Exception e) { - log.log(LogLevel.FATAL, "Failed to stop application '" + app.getName() + "': " + e.getMessage()); + log.log(Level.SEVERE, "Failed to stop application '" + app.getName() + "': " + e.getMessage()); e.printStackTrace(); return; } @@ -48,7 +48,7 @@ public class StandAloneClusterController { try{ myApp.start(); } catch (Exception e) { - log.log(LogLevel.FATAL, "Failed to start application '" + myApp.getName() + "': " + e.getMessage()); + log.log(Level.SEVERE, "Failed to start application '" + myApp.getName() + "': " + e.getMessage()); e.printStackTrace(); return; } @@ -56,7 +56,7 @@ public class StandAloneClusterController { try{ myApp.run(); } catch (Exception e) { - log.log(LogLevel.FATAL, "Application '" + myApp.getName() + "' runtime failure: " + e.getMessage()); + log.log(Level.SEVERE, "Application '" + myApp.getName() + "' runtime failure: " + e.getMessage()); e.printStackTrace(); } } diff --git a/clustercontroller-utils/pom.xml b/clustercontroller-utils/pom.xml index 0fed0c67eb5..bdf57a9caf0 100644 --- a/clustercontroller-utils/pom.xml +++ b/clustercontroller-utils/pom.xml @@ -30,7 +30,7 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> + <artifactId>vespajlib</artifactId> <version>${project.version}</version> <scope>provided</scope> </dependency> diff --git a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java index acc4782e5eb..953f1000236 100644 --- a/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java +++ b/clustercontroller-utils/src/main/java/com/yahoo/vespa/clustercontroller/utils/staterestapi/server/RestApiHandler.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.clustercontroller.utils.staterestapi.server; import com.google.common.util.concurrent.UncheckedTimeoutException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.time.TimeBudget; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequest; import com.yahoo.vespa.clustercontroller.utils.communication.http.HttpRequestHandler; @@ -103,7 +103,7 @@ public class RestApiHandler implements HttpRequestHandler { return new JsonHttpResult().setJson(jsonWriter.createJson(setResponse)); } } catch (OtherMasterException exception) { - logRequestException(request, exception, LogLevel.DEBUG); + logRequestException(request, exception, Level.FINE); JsonHttpResult result = new JsonHttpResult(); result.setHttpCode(307, "Temporary Redirect"); result.addHeader("Location", getMasterLocationUrl(request, exception.getHost(), exception.getPort())); @@ -130,7 +130,7 @@ public class RestApiHandler implements HttpRequestHandler { result.setJson(jsonWriter.createErrorJson(exception.getMessage())); return result; } catch (Exception exception) { - logRequestException(request, exception, LogLevel.ERROR); + logRequestException(request, exception, Level.SEVERE); JsonHttpResult result = new JsonHttpResult(); result.setHttpCode(500, "Failed to process request"); result.setJson(jsonWriter.createErrorJson(exception.getClass().getName() + ": " + exception.getMessage())); diff --git a/config-application-package/pom.xml b/config-application-package/pom.xml index 2cce4c01243..9deeeb64c8e 100644 --- a/config-application-package/pom.xml +++ b/config-application-package/pom.xml @@ -50,12 +50,6 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> <groupId>com.thaiopensource</groupId> <artifactId>jing</artifactId> <version>20091111</version> diff --git a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java index 23273f9f1bf..6a946e1ce75 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/OverrideProcessor.java @@ -4,7 +4,7 @@ package com.yahoo.config.application; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.XML; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -54,7 +54,7 @@ class OverrideProcessor implements PreProcessor { } public Document process(Document input) throws TransformerException { - log.log(LogLevel.DEBUG, "Preprocessing overrides with " + environment + "." + region); + log.log(Level.FINE, "Preprocessing overrides with " + environment + "." + region); Document ret = Xml.copyDocument(input); Element root = ret.getDocumentElement(); applyOverrides(root, Context.empty()); diff --git a/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java b/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java index 65120c3677c..1c5ea8510cb 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/PropertiesProcessor.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.XML; import org.w3c.dom.*; @@ -42,7 +42,7 @@ class PropertiesProcessor implements PreProcessor { //System.out.println("Found " + node.getNodeName() + ", " + node.getTextContent()); final String propertyName = node.getNodeName(); if (properties.containsKey(propertyName)) { - log.log(LogLevel.WARNING, "Duplicate definition for property '" + propertyName + "' detected"); + log.log(Level.WARNING, "Duplicate definition for property '" + propertyName + "' detected"); } properties.put(propertyName, node.getTextContent()); } diff --git a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java index 1cdb54a743c..c48a41083c7 100644 --- a/config-application-package/src/main/java/com/yahoo/config/application/Xml.java +++ b/config-application-package/src/main/java/com/yahoo/config/application/Xml.java @@ -3,7 +3,7 @@ package com.yahoo.config.application; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.io.reader.NamedReader; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.XML; import org.w3c.dom.Document; @@ -61,7 +61,7 @@ public class Xml { try { return factory.newDocumentBuilder(); } catch (ParserConfigurationException e) { - log.log(LogLevel.WARNING, "No XML parser available - " + e); + log.log(Level.WARNING, "No XML parser available - " + e); return null; } } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java index a8e1256e032..31349967462 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationFile.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.yolean.Exceptions; import com.yahoo.vespa.config.util.ConfigUtils; @@ -41,7 +41,7 @@ public class FilesApplicationFile extends ApplicationFile { @Override public ApplicationFile delete() { - log.log(LogLevel.DEBUG, "Delete " + file); + log.log(Level.FINE, "Delete " + file); if (file.isDirectory() && !listFiles().isEmpty()) { throw new RuntimeException("files. Can't delete, directory not empty: " + this + "(" + listFiles() + ")." + listFiles().size()); } @@ -154,7 +154,7 @@ public class FilesApplicationFile extends ApplicationFile { private void writeMetaFile(String data, String status) throws IOException { File metaDir = createMetaDir(); - log.log(LogLevel.DEBUG, "meta dir=" + metaDir); + log.log(Level.FINE, "meta dir=" + metaDir); File metaFile = new File(metaDir + "/" + getPath().getName()); if (status == null) { status = ApplicationFile.ContentStatusNew; @@ -174,7 +174,7 @@ public class FilesApplicationFile extends ApplicationFile { private File createMetaDir() { File metaDir = getMetaDir(); if (!metaDir.exists()) { - log.log(LogLevel.DEBUG, "Creating meta dir " + metaDir); + log.log(Level.FINE, "Creating meta dir " + metaDir); metaDir.mkdirs(); } return metaDir; @@ -188,7 +188,7 @@ public class FilesApplicationFile extends ApplicationFile { public MetaData getMetaData() { File metaDir = getMetaDir(); File metaFile = new File(metaDir + "/" + getPath().getName()); - log.log(LogLevel.DEBUG, "Getting metadata for " + metaFile); + log.log(Level.FINE, "Getting metadata for " + metaFile); if (metaFile.exists()) { try { return mapper.readValue(metaFile, MetaData.class); diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index e93246f49e7..2e4341f59ae 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -22,7 +22,7 @@ import com.yahoo.path.Path; import com.yahoo.io.HexDump; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionBuilder; @@ -282,7 +282,7 @@ public class FilesApplicationPackage implements ApplicationPackage { String dir = include.getAttribute(IncludeDirs.DIR); validateIncludeDir(dir); IncludeDirs.validateFilesInIncludedDir(dir, include.getParentNode(), this); - log.log(LogLevel.DEBUG, "Adding user include dir '" + dir + "'"); + log.log(Level.FINE, "Adding user include dir '" + dir + "'"); userIncludeDirs.add(dir); } @@ -458,7 +458,7 @@ public class FilesApplicationPackage implements ApplicationPackage { private void addAllDefsFromConfigDir(Map<ConfigDefinitionKey, UnparsedConfigDefinition> defs, File configDefsDir) { if (! configDefsDir.isDirectory()) return; - log.log(LogLevel.DEBUG, "Getting all config definitions from '" + configDefsDir + "'"); + log.log(Level.FINE, "Getting all config definitions from '" + configDefsDir + "'"); for (File def : configDefsDir.listFiles((File dir, String name) -> name.matches(".*\\.def"))) { String[] nv = def.getName().split("\\.def"); ConfigDefinitionKey key; @@ -473,10 +473,10 @@ public class FilesApplicationPackage implements ApplicationPackage { if (defs.containsKey(key)) { if (nv[0].contains(".")) { - log.log(LogLevel.INFO, "Two config definitions found for the same name and namespace: " + key + + log.log(Level.INFO, "Two config definitions found for the same name and namespace: " + key + ". The file '" + def + "' will take precedence"); } else { - log.log(LogLevel.INFO, "Two config definitions found for the same name and namespace: " + key + + log.log(Level.INFO, "Two config definitions found for the same name and namespace: " + key + ". Skipping '" + def + "', as it does not contain namespace in filename"); continue; // skip } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java index d5d890dba43..28f74b4adc0 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/SchemaValidators.java @@ -3,7 +3,7 @@ package com.yahoo.config.model.application.provider; import com.yahoo.component.Version; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.osgi.framework.Bundle; import org.xml.sax.SAXException; @@ -110,10 +110,10 @@ public class SchemaValidators { if (uris == null) throw new IllegalArgumentException("Could not find XML schemas "); File tmpDir = createTempDirectory(tmpBase.toPath(), "vespa").toFile(); - log.log(LogLevel.DEBUG, "Will save all XML schemas for " + vespaVersion + " to " + tmpDir); + log.log(Level.FINE, "Will save all XML schemas for " + vespaVersion + " to " + tmpDir); while (uris.hasMoreElements()) { URL u = uris.nextElement(); - log.log(LogLevel.DEBUG, "uri for resource 'schema'=" + u.toString()); + log.log(Level.FINE, "uri for resource 'schema'=" + u.toString()); // TODO: When is this the case? Remove? if ("jar".equals(u.getProtocol())) { JarURLConnection jarConnection = (JarURLConnection) u.openConnection(); @@ -127,7 +127,7 @@ public class SchemaValidators { jarFile.close(); } else if ("bundle".equals(u.getProtocol())) { Bundle bundle = getBundle(schemaValidatorClass); - log.log(LogLevel.DEBUG, "bundle=" + bundle); + log.log(Level.FINE, "bundle=" + bundle); // TODO: Hack to handle cases where bundle=null (which seems to always be the case with config-model-fat-amended.jar) if (bundle == null) { String pathPrefix = getDefaults().underVespaHome("share/vespa/schema/"); @@ -135,10 +135,10 @@ public class SchemaValidators { // Fallback to path without version if path with version does not exist if (! schemaPath.exists()) schemaPath = new File(pathPrefix); - log.log(LogLevel.DEBUG, "Using schemas found in " + schemaPath); + log.log(Level.FINE, "Using schemas found in " + schemaPath); copySchemas(schemaPath, tmpDir); } else { - log.log(LogLevel.DEBUG, String.format("Saving schemas for model bundle %s:%s", bundle.getSymbolicName(), bundle + log.log(Level.FINE, String.format("Saving schemas for model bundle %s:%s", bundle.getSymbolicName(), bundle .getVersion())); for (Enumeration<URL> entries = bundle.findEntries("schema", "*.rnc", true); entries.hasMoreElements(); ) { diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java index 872aa7fb56f..c930adfb115 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/StaticConfigDefinitionRepo.java @@ -3,7 +3,7 @@ package com.yahoo.config.model.application.provider; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.buildergen.ConfigDefinition; import com.yahoo.vespa.config.util.ConfigUtils; @@ -47,7 +47,7 @@ public class StaticConfigDefinitionRepo implements ConfigDefinitionRepo { ConfigDefinitionKey key = ConfigUtils.createConfigDefinitionKeyFromDefFile(def); addConfigDefinition(key, def); } catch (IOException e) { - log.log(LogLevel.WARNING, "Exception adding config definition " + def, e); + log.log(Level.WARNING, "Exception adding config definition " + def, e); } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 151a881eabe..4f7cf8575c4 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -77,6 +77,12 @@ public interface ModelContext { double defaultTermwiseLimit(); // TODO Revisit in May or June 2020 + double threadPoolSizeFactor(); + + // TODO Revisit in May or June 2020 + double queueSizeFactor(); + + // TODO Revisit in May or June 2020 double defaultSoftStartSeconds(); // TODO Revisit in May or June 2020 diff --git a/config-model/pom.xml b/config-model/pom.xml index 33f6657561c..897236f78e7 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -191,12 +191,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>statistics</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java index 4e9eb3bdb2e..454da0d657a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelRepo.java @@ -11,7 +11,7 @@ import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.graph.ModelGraphBuilder; import com.yahoo.config.model.graph.ModelNode; import com.yahoo.config.model.provision.HostsXmlProvisioner; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.XML; import com.yahoo.config.model.producer.AbstractConfigProducer; @@ -266,7 +266,7 @@ public class ConfigModelRepo implements ConfigModelRepoAdder, Serializable, Iter // TODO: Doctoring on the XML is the wrong level for this. We should be able to mark a model as default instead -Jon private static Element getImplicitAdmin(DeployState deployState) throws IOException, SAXException { String defaultAdminElement = deployState.isHosted() ? getImplicitAdminV4() : getImplicitAdminV2(); - log.log(LogLevel.DEBUG, "No <admin> defined, using " + defaultAdminElement); + log.log(Level.FINE, "No <admin> defined, using " + defaultAdminElement); return XmlHelper.getDocumentBuilder().parse(new InputSource(new StringReader(defaultAdminElement))).getDocumentElement(); } diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java index 4cd0c1815dd..a392a778709 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/XmlHelper.java @@ -3,7 +3,7 @@ package com.yahoo.config.model.builder.xml; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.XML; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -121,10 +121,10 @@ public final class XmlHelper { public static synchronized DocumentBuilder getDocumentBuilder() { try { DocumentBuilder docBuilder = factory.newDocumentBuilder(); - log.log(LogLevel.DEBUG, "XML parser now operational!"); + log.log(Level.FINE, "XML parser now operational!"); return docBuilder; } catch (ParserConfigurationException e) { - log.log(LogLevel.WARNING, "No XML parser available - " + e); + log.log(Level.WARNING, "No XML parser available - " + e); return null; } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 99225beba4f..d799af36c3b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -43,6 +43,8 @@ public class TestProperties implements ModelContext.Properties { private double topKProbability = 1.0; private double defaultTermwiseLimit = 1.0; private double softStartSeconds = 0.0; + private double threadPoolSizeFactor = 0.0; + private double queueSizeFactor = 0.0; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); private AthenzDomain athenzDomain; @@ -65,6 +67,16 @@ public class TestProperties implements ModelContext.Properties { @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override + public double threadPoolSizeFactor() { + return threadPoolSizeFactor; + } + + @Override + public double queueSizeFactor() { + return queueSizeFactor; + } + + @Override public double defaultSoftStartSeconds() { return softStartSeconds; } @@ -86,7 +98,15 @@ public class TestProperties implements ModelContext.Properties { this.softStartSeconds = softStartSeconds; return this; } + public TestProperties setThreadPoolSizeFactor(double threadPoolSizeFactor) { + this.threadPoolSizeFactor = threadPoolSizeFactor; + return this; + } + public TestProperties setQueueSizeFactor(double queueSizeFactor) { + this.queueSizeFactor = queueSizeFactor; + return this; + } public TestProperties setApplicationId(ApplicationId applicationId) { this.applicationId = applicationId; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java index cce5a7850a0..602e0c80d4b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AbstractConfigProducer.java @@ -6,7 +6,7 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.subscription.ConfigInstanceUtil; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; @@ -202,8 +202,8 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce found = parent.cascadeConfig(builder); boolean foundHere = builder.dispatchGetConfig(this); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "cascadeconfig in " + this + ", getting config " + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "cascadeconfig in " + this + ", getting config " + builder.getClass().getDeclaringClass().getName() + " for config id '" + configId + "' found here=" + foundHere); } found = found || foundHere; @@ -217,14 +217,14 @@ public abstract class AbstractConfigProducer<CHILD extends AbstractConfigProduce didApply = parent.addUserConfig(builder); } - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "User configs is: " + userConfigs.toString()); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "User configs is: " + userConfigs.toString()); } // TODO: What do we do with md5. Currently ignored for user configs? ConfigDefinitionKey key = new ConfigDefinitionKey(builder.getDefName(), builder.getDefNamespace()); if (userConfigs.get(key) != null) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Apply in " + configId); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Apply in " + configId); } applyUserConfig(builder, userConfigs.get(key)); didApply = true; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java index 6de7c985326..4011ce43841 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java @@ -41,6 +41,9 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement private final Map<Reference, TensorType> resolvedTypes = new HashMap<>(); + /** To avoid re-resolving diamond-shaped dependencies */ + private final Map<Reference, TensorType> globallyResolvedTypes; + /** For invocation loop detection */ private final Deque<Reference> currentResolutionCallStack; @@ -53,6 +56,7 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement this.currentResolutionCallStack = new ArrayDeque<>(); this.queryFeaturesNotDeclared = new TreeSet<>(); tensorsAreUsed = false; + globallyResolvedTypes = new HashMap<>(); } private MapEvaluationTypeContext(Map<String, ExpressionFunction> functions, @@ -60,12 +64,14 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement Map<Reference, TensorType> featureTypes, Deque<Reference> currentResolutionCallStack, SortedSet<Reference> queryFeaturesNotDeclared, - boolean tensorsAreUsed) { + boolean tensorsAreUsed, + Map<Reference, TensorType> globallyResolvedTypes) { super(functions, bindings); this.featureTypes.putAll(featureTypes); this.currentResolutionCallStack = currentResolutionCallStack; this.queryFeaturesNotDeclared = queryFeaturesNotDeclared; this.tensorsAreUsed = tensorsAreUsed; + this.globallyResolvedTypes = globallyResolvedTypes; } public void setType(Reference reference, TensorType type) { @@ -82,11 +88,25 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement resolvedTypes.clear(); } - @Override + private boolean referenceCanBeResolvedGlobally(Reference reference) { + Optional<ExpressionFunction> function = functionInvocation(reference); + return function.isPresent() && function.get().arguments().size() == 0; + // are there other cases we would like to resolve globally? + } + + @Override public TensorType getType(Reference reference) { // computeIfAbsent without concurrent modification due to resolve adding more resolved entries: + + boolean canBeResolvedGlobally = referenceCanBeResolvedGlobally(reference); + TensorType resolvedType = resolvedTypes.get(reference); - if (resolvedType != null) return resolvedType; + if (resolvedType == null && canBeResolvedGlobally) { + resolvedType = globallyResolvedTypes.get(reference); + } + if (resolvedType != null) { + return resolvedType; + } resolvedType = resolveType(reference); if (resolvedType == null) @@ -94,6 +114,11 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement resolvedTypes.put(reference, resolvedType); if (resolvedType.rank() > 0) tensorsAreUsed = true; + + if (canBeResolvedGlobally) { + globallyResolvedTypes.put(reference, resolvedType); + } + return resolvedType; } @@ -254,7 +279,8 @@ public class MapEvaluationTypeContext extends FunctionReferenceContext implement featureTypes, currentResolutionCallStack, queryFeaturesNotDeclared, - tensorsAreUsed); + tensorsAreUsed, + globallyResolvedTypes); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 23eb814de81..ea126123a25 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -680,11 +680,12 @@ public class RankProfile implements Cloneable { Map<String, RankingExpressionFunction> inlineFunctions = compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Collections.emptyMap(), expressionTransforms); + firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + // Function compiling second pass: compile all functions and insert previously compiled inline functions functions = compileFunctions(this::getFunctions, queryProfiles, featureTypes, importedModels, inlineFunctions, expressionTransforms); - firstPhaseRanking = compile(this.getFirstPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); - secondPhaseRanking = compile(this.getSecondPhaseRanking(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); } private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Value> constants) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java index 1a22b98fd9f..c3c10139684 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java @@ -200,9 +200,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (functions.isEmpty()) return; List<ExpressionFunction> functionExpressions = functions.values().stream().map(f -> f.function()).collect(Collectors.toList()); - Map<String, String> functionProperties = new LinkedHashMap<>(); - functionProperties.putAll(deriveFunctionProperties(functions, functionExpressions)); if (firstPhaseRanking != null) { functionProperties.putAll(firstPhaseRanking.getRankProperties(functionExpressions)); @@ -211,20 +209,30 @@ public class RawRankProfile implements RankProfilesConfig.Producer { functionProperties.putAll(secondPhaseRanking.getRankProperties(functionExpressions)); } + SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); + replaceFunctionSummaryFeatures(context); + + // First phase, second phase and summary features should add all required functions to the context. + // However, we need to add any functions not referenced in those anyway for model-evaluation. + deriveFunctionProperties(functions, functionExpressions, functionProperties); + for (Map.Entry<String, String> e : functionProperties.entrySet()) { rankProperties.add(new RankProfile.RankProperty(e.getKey(), e.getValue())); } - SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); - replaceFunctionSummaryFeatures(context); } - private Map<String, String> deriveFunctionProperties(Map<String, RankProfile.RankingExpressionFunction> functions, - List<ExpressionFunction> functionExpressions) { - SerializationContext context = new SerializationContext(functionExpressions); + private void deriveFunctionProperties(Map<String, RankProfile.RankingExpressionFunction> functions, + List<ExpressionFunction> functionExpressions, + Map<String, String> functionProperties) { + SerializationContext context = new SerializationContext(functionExpressions, null, functionProperties); for (Map.Entry<String, RankProfile.RankingExpressionFunction> e : functions.entrySet()) { + String propertyName = RankingExpression.propertyName(e.getKey()); + if (context.serializedFunctions().containsKey(propertyName)) { + continue; + } String expressionString = e.getValue().function().getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); - context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expressionString); + context.addFunctionSerialization(RankingExpression.propertyName(e.getKey()), expressionString); for (Map.Entry<String, TensorType> argumentType : e.getValue().function().argumentTypes().entrySet()) context.addArgumentTypeSerialization(e.getKey(), argumentType.getKey(), argumentType.getValue()); if (e.getValue().function().returnType().isPresent()) @@ -232,7 +240,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { // else if (e.getValue().function().arguments().isEmpty()) TODO: Enable this check when we resolve all types // throw new IllegalStateException("Type of function '" + e.getKey() + "' is not resolved"); } - return context.serializedFunctions(); + functionProperties.putAll(context.serializedFunctions()); } private void replaceFunctionSummaryFeatures(SerializationContext context) { @@ -241,9 +249,11 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (Iterator<ReferenceNode> i = summaryFeatures.iterator(); i.hasNext(); ) { ReferenceNode referenceNode = i.next(); // Is the feature a function? - if (context.getFunction(referenceNode.getName()) != null) { - context.addFunctionSerialization(RankingExpression.propertyName(referenceNode.getName()), - referenceNode.toString(new StringBuilder(), context, null, null).toString()); + ExpressionFunction function = context.getFunction(referenceNode.getName()); + if (function != null) { + String propertyName = RankingExpression.propertyName(referenceNode.getName()); + String expressionString = function.getBody().getRoot().toString(new StringBuilder(), context, null, null).toString(); + context.addFunctionSerialization(propertyName, expressionString); ReferenceNode newReferenceNode = new ReferenceNode("rankingExpression(" + referenceNode.getName() + ")", referenceNode.getArguments().expressions(), referenceNode.getOutput()); functionSummaryFeatures.put(referenceNode.getName(), newReferenceNode); i.remove(); // Will add the expanded one in next block diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 557a61ec211..292d4761d8a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -21,7 +21,7 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; /** * The parent node for all Host instances, and thus accessible @@ -120,7 +120,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { HostResource host = getExistingHost(spec).orElseGet(() -> addNewHost(spec)); retAllocatedHosts.put(host, spec.membership().orElse(null)); } - retAllocatedHosts.keySet().forEach(host -> log.log(DEBUG, () -> "Allocated host " + host.getHostname() + " with flavor " + host.getFlavor())); + retAllocatedHosts.keySet().forEach(host -> log.log(FINE, () -> "Allocated host " + host.getHostname() + " with flavor " + host.getFlavor())); return retAllocatedHosts; } @@ -131,7 +131,7 @@ public class HostSystem extends AbstractConfigProducer<Host> { if (hosts.isEmpty()) { return Optional.empty(); } else { - log.log(DEBUG, () -> "Found existing host resource for " + key.hostname() + " with flavor " + hosts.get(0).getFlavor()); + log.log(FINE, () -> "Found existing host resource for " + key.hostname() + " with flavor " + hosts.get(0).getFlavor()); return Optional.of(hosts.get(0)); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 6f2123f248c..b63497d64a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -30,7 +30,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.RankingConstants; @@ -357,7 +357,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri */ protected void checkId(String configId) { if ( ! id2producer.containsKey(configId)) { - log.log(LogLevel.DEBUG, "Invalid config id: " + configId); + log.log(Level.FINE, "Invalid config id: " + configId); } } @@ -382,7 +382,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri private static void populateConfigBuilder(Builder builder, ConfigProducer configProducer) { boolean found = configProducer.cascadeConfig(builder); boolean foundOverride = configProducer.addUserConfig(builder); - log.log(LogLevel.DEBUG, () -> "Trying to get config for " + builder.getClass().getDeclaringClass().getName() + + log.log(Level.FINE, () -> "Trying to get config for " + builder.getClass().getDeclaringClass().getName() + " for config id " + quote(configProducer.getConfigId()) + ", found=" + found + ", foundOverride=" + foundOverride); @@ -400,7 +400,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri ConfigInstance.Builder builder = resolveToBuilder(configKey); // TODO: remove if-statement, the builder can never be null. if (builder != null) { - log.log(LogLevel.DEBUG, () -> "Found builder for " + configKey); + log.log(Level.FINE, () -> "Found builder for " + configKey); ConfigPayload payload; InnerCNode innerCNode = targetDef != null ? targetDef.getCNode() : null; if (builder instanceof GenericConfig.GenericConfigBuilder) { @@ -429,14 +429,14 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri private ConfigPayload getConfigFromBuilder(ConfigInstance.Builder builder, InnerCNode targetDef) { try { ConfigInstance instance = InstanceResolver.resolveToInstance(builder, targetDef); - log.log(LogLevel.DEBUG, () -> "getConfigFromBuilder for builder " + builder.getClass().getName() + ", instance=" + instance); + log.log(Level.FINE, () -> "getConfigFromBuilder for builder " + builder.getClass().getName() + ", instance=" + instance); return ConfigPayload.fromInstance(instance); } catch (ConfigurationRuntimeException e) { // This can happen in cases where services ask for config that no longer exist before they have been able // to reconfigure themselves. This happens for instance whenever jdisc reconfigures itself until // ticket 6599572 is fixed. When that happens, consider propagating a full error rather than empty payload // back to the client. - log.log(LogLevel.INFO, "Error resolving instance for builder '" + builder.getClass().getName() + "', returning empty config: " + Exceptions.toMessageString(e)); + log.log(Level.INFO, "Error resolving instance for builder '" + builder.getClass().getName() + "', returning empty config: " + Exceptions.toMessageString(e)); return ConfigPayload.fromBuilder(new ConfigPayloadBuilder()); } } @@ -465,12 +465,12 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri final String builderName = fullClassName + "$Builder"; if (classLoader == null) { classLoader = getClass().getClassLoader(); - log.log(LogLevel.DEBUG, () -> "No producer found to get classloader from for " + fullClassName + ". Using default"); + log.log(Level.FINE, () -> "No producer found to get classloader from for " + fullClassName + ". Using default"); } try { clazz = classLoader.loadClass(builderName); } catch (ClassNotFoundException e) { - log.log(LogLevel.DEBUG, () -> "Tried to load " + builderName + ", not found, trying with generic builder"); + log.log(Level.FINE, () -> "Tried to load " + builderName + ", not found, trying with generic builder"); // TODO: Enable config compiler when configserver is using new API. // ConfigCompiler compiler = new LazyConfigCompiler(Files.createTempDir()); // return compiler.compile(targetDef.generateClass()).newInstance(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index b6f7ab4ff62..96dc312abd7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.io.IOUtils; import com.yahoo.log.InvalidLogFormatException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogMessage; import com.yahoo.yolean.Exceptions; import com.yahoo.system.ProcessExecuter; @@ -89,7 +89,7 @@ public class RankSetupValidator extends Validator { // Give up, don't say same error msg repeatedly deleteTempDir(tempDir); } - log.log(LogLevel.DEBUG, String.format("Validating %s for %s, %s took %s ms", + log.log(Level.FINE, String.format("Validating %s for %s, %s took %s ms", sdName, searchCluster, configId, @@ -154,7 +154,7 @@ public class RankSetupValidator extends Validator { private void validateWarn(Exception e, DeployLogger deployLogger) { String msg = "Unable to execute '"+ binaryName + "', validation of rank expressions will only take place when you start Vespa: " + Exceptions.toMessageString(e); - deployLogger.log(LogLevel.WARNING, msg); + deployLogger.log(Level.WARNING, msg); } private void validateFail(String output, SearchCluster sc, String sdName, DeployLogger deployLogger) { @@ -171,7 +171,7 @@ public class RankSetupValidator extends Validator { } } if (ignoreValidationErrors) { - deployLogger.log(LogLevel.WARNING, errMsg.append("(Continuing since ignoreValidationErrors flag is set.)").toString()); + deployLogger.log(Level.WARNING, errMsg.append("(Continuing since ignoreValidationErrors flag is set.)").toString()); } else { throw new IllegalArgumentException(errMsg.toString()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java index cb194e34bca..be85861ddb6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.builder; import com.yahoo.config.model.deploy.ConfigDefinitionStore; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.producer.UserConfigRepo; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.XML; import com.yahoo.vespa.config.*; import com.yahoo.vespa.model.builder.xml.dom.DomConfigPayloadBuilder; @@ -24,9 +24,9 @@ public class UserConfigBuilder { public static UserConfigRepo build(Element producerSpec, ConfigDefinitionStore configDefinitionStore, DeployLogger deployLogger) { final Map<ConfigDefinitionKey, ConfigPayloadBuilder> builderMap = new LinkedHashMap<>(); if (producerSpec == null) { - log.log(LogLevel.SPAM, "In getUserConfigs. producerSpec is null"); + log.log(Level.FINEST, "In getUserConfigs. producerSpec is null"); } - log.log(LogLevel.DEBUG, "getUserConfigs for " + producerSpec); + log.log(Level.FINE, "getUserConfigs for " + producerSpec); for (Element configE : XML.getChildren(producerSpec, "config")) { buildElement(configE, builderMap, configDefinitionStore, deployLogger); } @@ -40,13 +40,13 @@ public class UserConfigBuilder { Optional<ConfigDefinition> def = configDefinitionStore.getConfigDefinition(key); if ( ! def.isPresent()) { // TODO: Fail instead of warn - logger.log(LogLevel.WARNING, "Unable to find config definition '" + key.asFileName() + + logger.log(Level.WARNING, "Unable to find config definition '" + key.asFileName() + "'. Please ensure that the name is spelled correctly, and that the def file is included in a bundle."); } ConfigPayloadBuilder payloadBuilder = new DomConfigPayloadBuilder(def.orElse(null)).build(element); ConfigPayloadBuilder old = builderMap.get(key); if (old != null) { - logger.log(LogLevel.WARNING, "Multiple overrides for " + key + " found. Applying in the order they are discovered"); + logger.log(Level.WARNING, "Multiple overrides for " + key + " found. Applying in the order they are discovered"); old.override(payloadBuilder); } else { builderMap.put(key, payloadBuilder); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 1b0e04b50a8..706311dcdbf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -7,7 +7,7 @@ import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.text.XML; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.Configserver; @@ -53,7 +53,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { addLogForwarders(adminElement.child("logforwarding"), admin); if (adminElement.child("filedistribution") != null) { - deployState.getDeployLogger().log(LogLevel.WARNING, "'filedistribution' element is deprecated and ignored"); + deployState.getDeployLogger().log(Level.WARNING, "'filedistribution' element is deprecated and ignored"); } } @@ -117,7 +117,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { if (configserverE == null) { configserverE = XML.getChild(adminE, "adminserver"); } else { - deployState.getDeployLogger().log(LogLevel.INFO, "Specifying configserver without parent element configservers in services.xml is deprecated"); + deployState.getDeployLogger().log(Level.INFO, "Specifying configserver without parent element configservers in services.xml is deprecated"); } Configserver cfgs0 = new ConfigserverBuilder(0, configServerSpecs).build(deployState, configServers, configserverE); cfgs0.setProp("index", 0); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 804a5442608..a47c9fdb15b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -6,7 +6,7 @@ import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.ConfigServerSpec; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.admin.Admin; @@ -84,7 +84,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { createLogserver(deployState.getDeployLogger(), admin, hosts); } else { - context.getDeployLogger().log(LogLevel.INFO, "No container host available to use for running logserver"); + context.getDeployLogger().log(Level.INFO, "No container host available to use for running logserver"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index c9caca1831f..9ca5904a7c5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -8,7 +8,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.model.producer.UserConfigRepo; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.XML; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.Affinity; @@ -136,7 +136,7 @@ public class VespaDomBuilder extends VespaModelBuilder { UserConfigRepo userConfigs = UserConfigBuilder.build(producerSpec, deployState, deployState.getDeployLogger()); // TODO: must be made to work: //userConfigs.applyWarnings(child); - log.log(LogLevel.DEBUG, "Adding user configs " + userConfigs + " for " + producerSpec); + log.log(Level.FINE, "Adding user configs " + userConfigs + " for " + producerSpec); child.mergeUserConfigs(userConfigs); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index e59baf88422..29bb578a67b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.config.provision.Flavor; import com.yahoo.container.bundle.BundleInstantiationSpecification; +import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.prelude.fastsearch.FS4ResourcePool; import com.yahoo.search.config.QrStartConfig; @@ -15,7 +16,10 @@ import com.yahoo.vespa.model.container.component.Component; * * @author gjoranv */ -public final class ApplicationContainer extends Container implements QrStartConfig.Producer { +public final class ApplicationContainer extends Container implements + QrStartConfig.Producer, + ThreadpoolConfig.Producer +{ private static final String defaultHostedJVMArgs = "-XX:+UseOSErrorReporting -XX:+SuppressFatalErrorMessage"; @@ -73,4 +77,16 @@ public final class ApplicationContainer extends Container implements QrStartConf return (parent instanceof ContainerCluster) && (((ContainerCluster)parent).getDocproc() != null); } + @Override + public void getConfig(ThreadpoolConfig.Builder builder) { + if (! (parent instanceof ContainerCluster)) return; + if ((getHostResource() == null) || getHostResource().getFlavor().isEmpty()) return; + ContainerCluster containerCluster = (ContainerCluster) parent; + if (containerCluster.getThreadPoolSizeFactor() <= 0.0) return; + + NodeFlavorTuning flavorTuning = new NodeFlavorTuning(getHostResource().getFlavor().get()) + .setThreadPoolSizeFactor(containerCluster.getThreadPoolSizeFactor()) + .setQueueSizeFactor(containerCluster.getQueueSizeFactor()); + flavorTuning.getConfig(builder); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 833f6688fbc..b35b7562704 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -160,12 +160,18 @@ public abstract class ContainerCluster<CONTAINER extends Container> private String jvmGCOptions = null; private String environmentVars = null; + private final double threadPoolSizeFactor; + private final double queueSizeFactor; + public ContainerCluster(AbstractConfigProducer<?> parent, String subId, String name, DeployState deployState) { super(parent, subId); this.name = name; this.isHostedVespa = stateIsHosted(deployState); this.zone = (deployState != null) ? deployState.zone() : Zone.defaultZone(); + this.threadPoolSizeFactor = deployState.getProperties().threadPoolSizeFactor(); + this.queueSizeFactor = deployState.getProperties().queueSizeFactor(); + componentGroup = new ComponentGroup<>(this, "component"); addComponent(new StatisticsComponent()); @@ -186,6 +192,14 @@ public abstract class ContainerCluster<CONTAINER extends Container> addJaxProviders(); } + public double getThreadPoolSizeFactor() { + return threadPoolSizeFactor; + } + + public double getQueueSizeFactor() { + return queueSizeFactor; + } + public void setZone(Zone zone) { this.zone = zone; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java index 67938b36fd9..f9b50d0e641 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/NodeFlavorTuning.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.container; import com.yahoo.config.provision.Flavor; +import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; /** @@ -9,10 +10,26 @@ import com.yahoo.search.config.QrStartConfig; * * @author balder */ -public class NodeFlavorTuning implements QrStartConfig.Producer { +public class NodeFlavorTuning implements + QrStartConfig.Producer, + ThreadpoolConfig.Producer +{ private final Flavor flavor; + public NodeFlavorTuning setThreadPoolSizeFactor(double threadPoolSizeFactor) { + this.threadPoolSizeFactor = threadPoolSizeFactor; + return this; + } + + public NodeFlavorTuning setQueueSizeFactor(double queueSizeFactor) { + this.queueSizeFactor = queueSizeFactor; + return this; + } + + private double threadPoolSizeFactor = 8.0; + private double queueSizeFactor = 8.0; + NodeFlavorTuning(Flavor flavor) { this.flavor = flavor; } @@ -22,4 +39,15 @@ public class NodeFlavorTuning implements QrStartConfig.Producer { builder.jvm.availableProcessors(Math.max(2, (int)Math.ceil(flavor.getMinCpuCores()))); } + @Override + public void getConfig(ThreadpoolConfig.Builder builder) { + // Controls max number of concurrent requests per container + int workerThreads = Math.max(2, (int)Math.ceil(flavor.getMinCpuCores() * threadPoolSizeFactor)); + builder.maxthreads(workerThreads); + + // This controls your burst handling capability. + // 0 => No extra burst handling beyond you max concurrent requests (maxthreads). + // N => N times max concurrent requests as a buffer for handling bursts + builder.queueSize((int)(workerThreads * queueSizeFactor)); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index f0c374b398f..e380588f16e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -6,7 +6,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.content.StorDistributionConfig; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; @@ -209,7 +209,7 @@ public class StorageGroup { if (group.isPresent() && nodes.isPresent()) throw new IllegalStateException("Both group and nodes exists, only one of these tags is legal"); if (group.isPresent() && (group.get().stringAttribute("name") != null || group.get().integerAttribute("distribution-key") != null)) - deployState.getDeployLogger().log(LogLevel.INFO, "'distribution-key' attribute on a content cluster's root group is ignored"); + deployState.getDeployLogger().log(Level.INFO, "'distribution-key' attribute on a content cluster's root group is ignored"); GroupBuilder groupBuilder = collectGroup(owner.isHosted(), group, nodes, null, null); StorageGroup storageGroup = (owner.isHosted()) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java index 46d41b1e0b0..6e6a4d5ac8f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.utils; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.producer.UserConfigRepo; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.*; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.config.ConfigDefinition.DefaultValued; @@ -84,7 +84,7 @@ public class FileSender implements Serializable { ConfigDefinition configDefinition = builder.getConfigDefinition(); if (configDefinition == null) { // TODO: throw new IllegalArgumentException("Not able to find config definition for " + builder); - logger.log(LogLevel.WARNING, "Not able to find config definition for " + key + ". Will not send files for this config"); + logger.log(Level.WARNING, "Not able to find config definition for " + key + ". Will not send files for this config"); return; } // Inspect fields at this level diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java index 5c1134f928c..e4ca83640e9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java @@ -50,10 +50,10 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(s)).configProperties(); - assertEquals("(rankingExpression(sin).rankingScript,x * x)", - testRankProperties.get(0).toString()); assertEquals("(rankingExpression(sin@).rankingScript,2 * 2)", - censorBindingHash(testRankProperties.get(1).toString())); + censorBindingHash(testRankProperties.get(0).toString())); + assertEquals("(rankingExpression(sin).rankingScript,x * x)", + testRankProperties.get(1).toString()); assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", censorBindingHash(testRankProperties.get(2).toString())); } @@ -94,27 +94,26 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(s)).configProperties(); + assertEquals("(rankingExpression(tan@).rankingScript,2 * 2)", + censorBindingHash(testRankProperties.get(0).toString())); + assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(sin@).rankingScript,rankingExpression(cos@))", + censorBindingHash(testRankProperties.get(2).toString())); assertEquals("(rankingExpression(tan).rankingScript,x * x)", - testRankProperties.get(0).toString()); + testRankProperties.get(3).toString()); assertEquals("(rankingExpression(tan@).rankingScript,x * x)", - censorBindingHash(testRankProperties.get(1).toString())); - assertEquals("(rankingExpression(cos).rankingScript,rankingExpression(tan@))", - censorBindingHash(testRankProperties.get(2).toString())); - assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", - censorBindingHash(testRankProperties.get(3).toString())); - assertEquals("(rankingExpression(sin).rankingScript,rankingExpression(cos@))", censorBindingHash(testRankProperties.get(4).toString())); - assertEquals("(rankingExpression(tan@).rankingScript,2 * 2)", + assertEquals("(rankingExpression(cos).rankingScript,rankingExpression(tan@))", censorBindingHash(testRankProperties.get(5).toString())); assertEquals("(rankingExpression(cos@).rankingScript,rankingExpression(tan@))", - censorBindingHash(testRankProperties.get(6).toString())); - assertEquals("(rankingExpression(sin@).rankingScript,rankingExpression(cos@))", + censorBindingHash(testRankProperties.get(6).toString())); + assertEquals("(rankingExpression(sin).rankingScript,rankingExpression(cos@))", censorBindingHash(testRankProperties.get(7).toString())); assertEquals("(vespa.rank.firstphase,rankingExpression(sin@))", censorBindingHash(testRankProperties.get(8).toString())); } - @Test public void testFunctionShadowingArguments() throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); @@ -144,12 +143,12 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { new QueryProfileRegistry(), new ImportedMlModels(), new AttributeFields(s)).configProperties(); - assertEquals("(rankingExpression(sin).rankingScript,x * x)", - testRankProperties.get(0).toString()); assertEquals("(rankingExpression(sin@).rankingScript,4.0 * 4.0)", - censorBindingHash(testRankProperties.get(1).toString())); + censorBindingHash(testRankProperties.get(0).toString())); assertEquals("(rankingExpression(sin@).rankingScript,cos(5.0) * cos(5.0))", - censorBindingHash(testRankProperties.get(2).toString())); + censorBindingHash(testRankProperties.get(1).toString())); + assertEquals("(rankingExpression(sin).rankingScript,x * x)", + testRankProperties.get(2).toString()); assertEquals("(vespa.rank.firstphase,rankingExpression(firstphase))", censorBindingHash(testRankProperties.get(3).toString())); assertEquals("(rankingExpression(firstphase).rankingScript,cos(rankingExpression(sin@)) + rankingExpression(sin@))", @@ -208,17 +207,17 @@ public class RankingExpressionShadowingTestCase extends SchemaTestCase { queryProfiles, new ImportedMlModels(), new AttributeFields(s)).configProperties(); - assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))", - testRankProperties.get(0).toString()); assertEquals("(rankingExpression(relu@).rankingScript,max(1.0,reduce(query(q) * constant(W_hidden), sum, input) + constant(b_input)))", - censorBindingHash(testRankProperties.get(1).toString())); + censorBindingHash(testRankProperties.get(0).toString())); assertEquals("(rankingExpression(hidden_layer).rankingScript,rankingExpression(relu@))", - censorBindingHash(testRankProperties.get(2).toString())); + censorBindingHash(testRankProperties.get(1).toString())); assertEquals("(rankingExpression(hidden_layer).type,tensor(x[]))", - censorBindingHash(testRankProperties.get(3).toString())); + censorBindingHash(testRankProperties.get(2).toString())); assertEquals("(rankingExpression(final_layer).rankingScript,sigmoid(reduce(rankingExpression(hidden_layer) * constant(W_final), sum, hidden) + constant(b_final)))", - testRankProperties.get(4).toString()); + testRankProperties.get(3).toString()); assertEquals("(rankingExpression(final_layer).type,tensor(x[]))", + testRankProperties.get(4).toString()); + assertEquals("(rankingExpression(relu).rankingScript,max(1.0,x))", testRankProperties.get(5).toString()); assertEquals("(vespa.rank.secondphase,rankingExpression(secondphase))", testRankProperties.get(6).toString()); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java index 0cd6674751e..e6616ce0dd1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java @@ -41,6 +41,14 @@ class RankProfileSearchFixture { private Search search; private Map<String, RankProfile> compiledRankProfiles = new HashMap<>(); + public RankProfileRegistry getRankProfileRegistry() { + return rankProfileRegistry; + } + + public QueryProfileRegistry getQueryProfileRegistry() { + return queryProfileRegistry; + } + RankProfileSearchFixture(String rankProfiles) throws ParseException { this(MockApplicationPackage.createEmpty(), new QueryProfileRegistry(), rankProfiles); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java index a64a964727c..680f2dd9659 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorFlowTestCase.java @@ -319,7 +319,7 @@ public class RankingExpressionWithTensorFlowTestCase { @Test public void testFunctionGeneration() { final String name = "mnist_saved"; - final String expression = "join(join(reduce(join(join(join(reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String expression = "join(reduce(join(join(join(reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b))"; final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; final String functionExpression2 = "join(reduce(join(join(join(0.009999999776482582, imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; @@ -349,7 +349,7 @@ public class RankingExpressionWithTensorFlowTestCase { " rank-profile my_profile_child inherits my_profile {\n" + " }"; - final String expression = "join(join(reduce(join(join(join(reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b)), tensor(d0[1])(1.0), f(a,b)(a * b))"; + final String expression = "join(reduce(join(join(join(reduce(constant(" + name + "_dnn_hidden2_Const), sum, d2), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden2_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(" + name + "_dnn_outputs_bias_read), f(a,b)(a + b))"; final String functionExpression1 = "join(reduce(join(reduce(rename(input, (d0, d1), (d0, d4)), sum, d0), constant(" + name + "_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(" + name + "_dnn_hidden1_bias_read), f(a,b)(a + b))"; final String functionExpression2 = "join(reduce(join(join(join(0.009999999776482582, imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(a * b)), imported_ml_function_" + name + "_dnn_hidden1_add, f(a,b)(max(a,b))), constant(" + name + "_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(" + name + "_dnn_hidden2_bias_read), f(a,b)(a + b))"; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java index b3eda9b7e13..1567a4c3b5e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java @@ -44,20 +44,20 @@ public class RankingExpressionsTestCase extends SchemaTestCase { new AttributeFields(search)).configProperties(); assertEquals(6, rankProperties.size()); - assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(0).getFirst()); - assertEquals("var1 * var2 + 890", rankProperties.get(0).getSecond()); + assertEquals("rankingExpression(titlematch$).rankingScript", rankProperties.get(2).getFirst()); + assertEquals("var1 * var2 + 890", rankProperties.get(2).getSecond()); - assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(1).getFirst()); - assertEquals("78 + closeness(distance)", rankProperties.get(1).getSecond()); + assertEquals("rankingExpression(artistmatch).rankingScript", rankProperties.get(3).getFirst()); + assertEquals("78 + closeness(distance)", rankProperties.get(3).getSecond()); assertEquals("rankingExpression(firstphase).rankingScript", rankProperties.get(5).getFirst()); assertEquals("0.8 + 0.2 * rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c) + 0.8 * rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6) * closeness(distance)", rankProperties.get(5).getSecond()); - assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(3).getFirst()); - assertEquals("7 * 8 + 890", rankProperties.get(3).getSecond()); + assertEquals("rankingExpression(titlematch$@c7e4c2d0e6d9f2a1.1d4ed08e56cce2e6).rankingScript", rankProperties.get(1).getFirst()); + assertEquals("7 * 8 + 890", rankProperties.get(1).getSecond()); - assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(2).getFirst()); - assertEquals("4 * 5 + 890", rankProperties.get(2).getSecond()); + assertEquals("rankingExpression(titlematch$@126063073eb2deb.ab95cd69909927c).rankingScript", rankProperties.get(0).getFirst()); + assertEquals("4 * 5 + 890", rankProperties.get(0).getSecond()); } @Test(expected = IllegalArgumentException.class) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 33cf0635349..0f94df80421 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -9,9 +9,11 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockRoot; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.container.handler.ThreadpoolConfig; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.Host; @@ -164,6 +166,8 @@ public class ContainerClusterTest { cluster.getConfig(tpBuilder); ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(10, threadpoolConfig.maxthreads()); + assertEquals(0, threadpoolConfig.queueSize()); + assertEquals(0, threadpoolConfig.softStartSeconds(), 0); } @Test @@ -213,10 +217,44 @@ public class ContainerClusterTest { cluster.getConfig(tpBuilder); ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(500, threadpoolConfig.maxthreads()); + assertEquals(0, threadpoolConfig.queueSize()); assertEquals(300.0, threadpoolConfig.softStartSeconds(), 0.0); } @Test + public void requireThatPoolAndQueueCanNotBeControlledByPropertiesWhenNoFlavor() { + DeployState state = new DeployState.Builder().properties(new TestProperties() + .setThreadPoolSizeFactor(8.5) + .setQueueSizeFactor(13.3)) + .build(); + MockRoot root = new MockRoot("foo", state); + ApplicationContainerCluster cluster = createContainerCluster(root, false); + addContainer(root.deployLogger(), cluster, "c1", "host-c1"); + + ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); + cluster.getConfig(tpBuilder); + ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); + assertEquals(500, threadpoolConfig.maxthreads()); + assertEquals(0, threadpoolConfig.queueSize()); + assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0); + } + + @Test + public void requireThatPoolAndQueueCanBeControlledByPropertiesAndFlavor() { + FlavorsConfig.Flavor.Builder flavorBuilder = new FlavorsConfig.Flavor.Builder().name("my_flavor").minCpuCores(3); + NodeFlavorTuning nodeFlavorTuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder))) + .setThreadPoolSizeFactor(13.3) + .setQueueSizeFactor(17.5); + + ThreadpoolConfig.Builder tpBuilder = new ThreadpoolConfig.Builder(); + nodeFlavorTuning.getConfig(tpBuilder); + ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); + assertEquals(40, threadpoolConfig.maxthreads()); + assertEquals(700, threadpoolConfig.queueSize()); + assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0); + } + + @Test public void requireThatDefaultThreadPoolConfigIsSane() { MockRoot root = new MockRoot("foo"); ApplicationContainerCluster cluster = createContainerCluster(root, false); @@ -226,6 +264,7 @@ public class ContainerClusterTest { cluster.getConfig(tpBuilder); ThreadpoolConfig threadpoolConfig = new ThreadpoolConfig(tpBuilder); assertEquals(500, threadpoolConfig.maxthreads()); + assertEquals(0, threadpoolConfig.queueSize()); assertEquals(0.0, threadpoolConfig.softStartSeconds(), 0.0); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java index ca84eb5eed7..57dbf132883 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/MlModelsTest.java @@ -64,10 +64,10 @@ public class MlModelsTest { private final String testProfile = "rankingExpression(input).rankingScript: attribute(argument)\n" + "rankingExpression(input).type: tensor<float>(d0[],d1[784])\n" + - "rankingExpression(Placeholder).rankingScript: attribute(argument)\n" + - "rankingExpression(Placeholder).type: tensor<float>(d0[],d1[784])\n" + "rankingExpression(imported_ml_function_mnist_saved_dnn_hidden1_add).rankingScript: join(reduce(join(rename(rankingExpression(input), (d0, d1), (d0, d4)), constant(mnist_saved_dnn_hidden1_weights_read), f(a,b)(a * b)), sum, d4), constant(mnist_saved_dnn_hidden1_bias_read), f(a,b)(a + b))\n" + "rankingExpression(mnist_tensorflow).rankingScript: join(reduce(join(map(join(reduce(join(join(join(0.009999999776482582, rankingExpression(imported_ml_function_mnist_saved_dnn_hidden1_add), f(a,b)(a * b)), rankingExpression(imported_ml_function_mnist_saved_dnn_hidden1_add), f(a,b)(max(a,b))), constant(mnist_saved_dnn_hidden2_weights_read), f(a,b)(a * b)), sum, d3), constant(mnist_saved_dnn_hidden2_bias_read), f(a,b)(a + b)), f(a)(1.0507009873554805 * if (a >= 0, a, 1.6732632423543772 * (exp(a) - 1)))), constant(mnist_saved_dnn_outputs_weights_read), f(a,b)(a * b)), sum, d2), constant(mnist_saved_dnn_outputs_bias_read), f(a,b)(a + b))\n" + + "rankingExpression(Placeholder).rankingScript: attribute(argument)\n" + + "rankingExpression(Placeholder).type: tensor<float>(d0[],d1[784])\n" + "rankingExpression(mnist_softmax_tensorflow).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_saved_layer_Variable_read), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_saved_layer_Variable_1_read), f(a,b)(a + b))\n" + "rankingExpression(mnist_softmax_onnx).rankingScript: join(reduce(join(rename(rankingExpression(Placeholder), (d0, d1), (d0, d2)), constant(mnist_softmax_Variable), f(a,b)(a * b)), sum, d2), constant(mnist_softmax_Variable_1), f(a,b)(a + b))\n" + "rankingExpression(my_xgboost).rankingScript: if (f29 < -0.1234567, if (!(f56 >= -0.242398), 1.71218, -1.70044), if (f109 < 0.8723473, -1.94071, 1.85965)) + if (!(f60 >= -0.482947), if (f29 < -4.2387498, 0.784718, -0.96853), -6.23624)\n" + diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index f175af3e2f1..692c941877b 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -47,6 +47,14 @@ public final class ApplicationId implements Comparable<ApplicationId> { return new Builder().tenant(parts[0]).applicationName(parts[1]).instanceName(parts[2]).build(); } + public static ApplicationId fromFullString(String idString) { + String[] parts = idString.split("\\."); + if (parts.length < 3) + throw new IllegalArgumentException("Application ids must be on the form tenant.application.instance, but was " + idString); + + return new Builder().tenant(parts[0]).applicationName(parts[1]).instanceName(parts[2]).build(); + } + @Override public int hashCode() { return stringValue.hashCode(); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index 09ee25fb437..f723575c342 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -40,37 +40,6 @@ public final class Capacity { ") must be divisible by the number of groups (" + resources.groups() + ")"); } - /** Returns the number of nodes requested */ - @Deprecated // TODO: Remove after April 2020 - public int nodeCount() { return min.nodes(); } - - /** Returns the number of nodes requested (across all groups), or 0 if not specified */ - @Deprecated // TODO: Remove after April 2020 - public int nodes() { return min.nodes(); } - - /** Returns the number of groups requested, or 0 if not specified */ - @Deprecated // TODO: Remove after April 2020 - public int groups() { return min.groups(); } - - /** - * The node flavor requested, or empty if no legacy flavor name has been used. - * This may be satisfied by the requested flavor or a suitable replacement. - * - * @deprecated use nodeResources instead - */ - @Deprecated // TODO: Remove after March 2020 - public Optional<String> flavor() { - if (nodeResources().isEmpty()) return Optional.empty(); - return Optional.of(min.nodeResources().toString()); - } - - /** Returns the resources requested for each node, or empty to leave this decision to provisioning */ - @Deprecated // TODO: Remove after March 2020 - public Optional<NodeResources> nodeResources() { - if (min.nodeResources() == NodeResources.unspecified) return Optional.empty(); - return Optional.of(min.nodeResources()); - } - public ClusterResources minResources() { return min; } public ClusterResources maxResources() { return max; } @@ -103,7 +72,12 @@ public final class Capacity { /** Create a non-required, failable capacity request */ public static Capacity from(ClusterResources resources) { - return from(resources, false, true); + return from(resources, resources); + } + + /** Create a non-required, failable capacity request */ + public static Capacity from(ClusterResources min, ClusterResources max) { + return from(min, max, false, true); } public static Capacity from(ClusterResources resources, boolean required, boolean canFail) { @@ -114,23 +88,6 @@ public final class Capacity { return new Capacity(min, max, required, canFail, NodeType.tenant); } - /** Create a non-required, failable capacity request */ - @Deprecated // TODO: Remove after April 2020 - public static Capacity fromCount(int nodes, NodeResources resources) { - return fromCount(nodes, resources, false, true); - } - - @Deprecated // TODO: Remove after April 2020 - public static Capacity fromCount(int nodes, NodeResources resources, boolean required, boolean canFail) { - return fromCount(nodes, Optional.of(resources), required, canFail); - } - - @Deprecated // TODO: Remove after April 2020 - public static Capacity fromCount(int nodes, Optional<NodeResources> resources, boolean required, boolean canFail) { - return from(new ClusterResources(nodes, 0, resources.orElse(NodeResources.unspecified)), - required, canFail, NodeType.tenant); - } - /** Creates this from a node type */ public static Capacity fromRequiredNodeType(NodeType type) { return from(new ClusterResources(0, 0, NodeResources.unspecified), true, false, type); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java index 97549e851ad..3a230c89732 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterSpec.java @@ -29,7 +29,7 @@ public final class ClusterSpec { this.type = type; this.id = id; this.groupId = groupId; - this.vespaVersion = vespaVersion; + this.vespaVersion = Objects.requireNonNull(vespaVersion); this.exclusive = exclusive; // TODO(mpolden): Require combinedId to always be present for type combined after April 2020 if (type != Type.combined && combinedId.isPresent()) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java index e308d631442..2a1528f5368 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Provisioner.java @@ -24,9 +24,6 @@ public interface Provisioner { */ List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger); - @Deprecated // TODO: Remove after April 2020 - List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger); - /** * Activates the allocation of nodes to this application captured in the hosts argument. * diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index 0a36f06a1b9..a19d5809660 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -12,7 +12,7 @@ import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.TargetWatcher; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.JRTMethods; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; @@ -50,7 +50,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer public void run() { try { Acceptor acceptor = supervisor.listen(spec); - log.log(LogLevel.DEBUG, "Ready for requests on " + spec); + log.log(Level.FINE, "Ready for requests on " + spec); supervisor.transport().join(); acceptor.shutdown().join(); } catch (ListenFailedException e) { @@ -247,13 +247,13 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer private void dispatchRpcRequest(Request request, Runnable handler) { request.detach(); - log.log(LogLevel.SPAM, () -> String.format("Dispatching RPC request %s", requestLogId(request))); + log.log(Level.FINEST, () -> String.format("Dispatching RPC request %s", requestLogId(request))); rpcExecutor.execute(() -> { try { - log.log(LogLevel.SPAM, () -> String.format("Executing RPC request %s.", requestLogId(request))); + log.log(Level.FINEST, () -> String.format("Executing RPC request %s.", requestLogId(request))); handler.run(); } catch (Exception e) { - log.log(LogLevel.WARNING, + log.log(Level.WARNING, String.format("Exception thrown during execution of RPC request %s: %s", requestLogId(request), e.getMessage()), e); } }); @@ -270,21 +270,21 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer */ private void getConfigImpl(JRTServerConfigRequest request) { request.getRequestTrace().trace(TRACELEVEL, "Config proxy getConfig()"); - log.log(LogLevel.DEBUG, () ->"getConfig: " + request.getShortDescription() + ",configmd5=" + request.getRequestConfigMd5()); + log.log(Level.FINE, () ->"getConfig: " + request.getShortDescription() + ",configmd5=" + request.getRequestConfigMd5()); if (!request.validateParameters()) { // Error code is set in verifyParameters if parameters are not OK. - log.log(LogLevel.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); + log.log(Level.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); returnErrorResponse(request, request.errorCode(), "Parameters for request " + request.getShortDescription() + " did not validate: " + request.errorMessage()); return; } try { RawConfig config = proxyServer.resolveConfig(request); if (config == null) { - log.log(LogLevel.SPAM, () -> "No config received yet for " + request.getShortDescription() + ", not sending response"); + log.log(Level.FINEST, () -> "No config received yet for " + request.getShortDescription() + ", not sending response"); } else if (ProxyServer.configOrGenerationHasChanged(config, request)) { returnOkResponse(request, config); } else { - log.log(LogLevel.SPAM, "No new config for " + request.getShortDescription() + ", not sending response"); + log.log(Level.FINEST, "No new config for " + request.getShortDescription() + ", not sending response"); } } catch (Exception e) { e.printStackTrace(); @@ -335,12 +335,12 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer */ @Override public void notifyTargetInvalid(Target target) { - log.log(LogLevel.DEBUG, () -> "Target invalid " + target); + log.log(Level.FINE, () -> "Target invalid " + target); for (Iterator<DelayedResponse> it = proxyServer.delayedResponses().responses().iterator(); it.hasNext(); ) { DelayedResponse delayed = it.next(); JRTServerConfigRequest request = delayed.getRequest(); if (request.getRequest().target().equals(target)) { - log.log(LogLevel.DEBUG, () -> "Removing " + request.getShortDescription()); + log.log(Level.FINE, () -> "Removing " + request.getShortDescription()); it.remove(); } } @@ -351,9 +351,9 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer public void returnOkResponse(JRTServerConfigRequest request, RawConfig config) { request.getRequestTrace().trace(TRACELEVEL, "Config proxy returnOkResponse()"); request.addOkResponse(config.getPayload(), config.getGeneration(), config.isInternalRedeploy(), config.getConfigMd5()); - log.log(LogLevel.DEBUG, () -> "Return response: " + request.getShortDescription() + ",configMd5=" + config.getConfigMd5() + + log.log(Level.FINE, () -> "Return response: " + request.getShortDescription() + ",configMd5=" + config.getConfigMd5() + ",generation=" + config.getGeneration()); - log.log(LogLevel.SPAM, () -> "Config payload in response for " + request.getShortDescription() + ":" + config.getPayload()); + log.log(Level.FINEST, () -> "Config payload in response for " + request.getShortDescription() + ":" + config.getPayload()); // TODO Catch exception for now, since the request might have been returned in CheckDelayedResponse @@ -361,7 +361,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher, RpcServer try { request.getRequest().returnRequest(); } catch (IllegalStateException e) { - log.log(LogLevel.DEBUG, () -> "Something bad happened when sending response for '" + request.getShortDescription() + "':" + e.getMessage()); + log.log(Level.FINE, () -> "Something bad happened when sending response for '" + request.getShortDescription() + "':" + e.getMessage()); } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java index 9fb78e7e812..e9110a0bc78 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigSourceClient.java @@ -24,8 +24,6 @@ interface ConfigSourceClient { List<String> getSourceConnections(); - void updateSubscribers(RawConfig config); - DelayedResponses delayedResponses(); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java index ac0c036774b..317606ac3ef 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/DelayedResponseHandler.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.yolean.Exceptions; import com.yahoo.vespa.config.ConfigCacheKey; @@ -38,7 +38,7 @@ public class DelayedResponseHandler implements Runnable { void checkDelayedResponses() { try { long start = System.currentTimeMillis(); - log.log(LogLevel.SPAM, () -> "Running DelayedResponseHandler. There are " + delayedResponses.size() + + log.log(Level.FINEST, () -> "Running DelayedResponseHandler. There are " + delayedResponses.size() + " delayed responses. First one is " + delayedResponses.responses().peek()); DelayedResponse response; AtomicInteger i = new AtomicInteger(0); @@ -50,14 +50,14 @@ public class DelayedResponseHandler implements Runnable { rpcServer.returnOkResponse(request, config); i.incrementAndGet(); } else { - log.log(LogLevel.WARNING, "Timed out (timeout " + request.getTimeout() + ") getting config " + + log.log(Level.WARNING, "Timed out (timeout " + request.getTimeout() + ") getting config " + request.getConfigKey() + ", will retry"); } } - log.log(LogLevel.SPAM, () -> "Finished running DelayedResponseHandler. " + i.get() + " delayed responses sent in " + + log.log(Level.FINEST, () -> "Finished running DelayedResponseHandler. " + i.get() + " delayed responses sent in " + (System.currentTimeMillis() - start) + " ms"); } catch (Exception e) { // To avoid thread throwing exception and executor never running this again - log.log(LogLevel.WARNING, "Got exception in DelayedResponseHandler: " + Exceptions.toMessageString(e)); + log.log(Level.WARNING, "Got exception in DelayedResponseHandler: " + Exceptions.toMessageString(e)); } catch (Throwable e) { com.yahoo.protect.Process.logAndDie("Got error in DelayedResponseHandler, exiting: " + Exceptions.toMessageString(e)); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java index 91bb669f396..ac6fac3b16c 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCache.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.proxy; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigCacheKey; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.RawConfig; @@ -50,7 +50,7 @@ public class MemoryCache { return; } - log.log(LogLevel.DEBUG, () -> "Putting '" + config + "' into memory cache"); + log.log(Level.FINE, () -> "Putting '" + config + "' into memory cache"); cache.put(new ConfigCacheKey(config.getKey(), config.getDefMd5()), config); } @@ -78,7 +78,7 @@ public class MemoryCache { String dumpCacheToDisk(String path, MemoryCache cache) { if (path == null || path.isEmpty()) { path = DEFAULT_DUMP_DIR; - log.log(LogLevel.INFO, "dumpCache. No path or empty path. Using '" + path + "'"); + log.log(Level.INFO, "dumpCache. No path or empty path. Using '" + path + "'"); } if (path.endsWith("/")) { path = path.substring(0, path.length() - 1); @@ -86,7 +86,7 @@ public class MemoryCache { File dir = new File(path); if ( ! dir.exists()) { - log.log(LogLevel.INFO, dir.getAbsolutePath() + " does not exist, creating it"); + log.log(Level.INFO, dir.getAbsolutePath() + " does not exist, creating it"); try { Files.createDirectory(dir.toPath()); } catch (IOException e) { @@ -101,7 +101,7 @@ public class MemoryCache { return "Not able to write to '" + dir.getAbsolutePath() + "'"; } - log.log(LogLevel.INFO, "Dumping cache to '" + dir.getAbsolutePath() + "'"); + log.log(Level.INFO, "Dumping cache to '" + dir.getAbsolutePath() + "'"); for (RawConfig config : cache.values()) { writeConfigToFile(config, path); } @@ -113,12 +113,12 @@ public class MemoryCache { Writer writer = null; try { filename = path + File.separator + createCacheFileName(config); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Writing '" + config.getKey() + "' to '" + filename + "'"); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Writing '" + config.getKey() + "' to '" + filename + "'"); } final Payload payload = config.getPayload(); long protocolVersion = 3; - log.log(LogLevel.DEBUG, "Writing config '" + config + "' to file '" + filename + "' with protocol version " + protocolVersion); + log.log(Level.FINE, "Writing config '" + config + "' to file '" + filename + "' with protocol version " + protocolVersion); writer = IOUtils.createWriter(filename, "UTF-8", false); // First three lines are meta-data about config as comment lines, fourth line is empty @@ -131,7 +131,7 @@ public class MemoryCache { writer.write("\n"); writer.close(); } catch (IOException e) { - log.log(LogLevel.WARNING, "Could not write to file '" + filename + "'"); + log.log(Level.WARNING, "Could not write to file '" + filename + "'"); } finally { if (writer != null) { try { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java index 51446882025..2e55b9a6f86 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/MemoryCacheConfigClient.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.*; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; @@ -32,11 +32,11 @@ class MemoryCacheConfigClient implements ConfigSourceClient { */ @Override public RawConfig getConfig(RawConfig input, JRTServerConfigRequest request) { - log.log(LogLevel.DEBUG, () -> "Getting config from cache"); + log.log(Level.FINE, () -> "Getting config from cache"); ConfigKey<?> key = input.getKey(); RawConfig cached = cache.get(new ConfigCacheKey(key, input.getDefMd5())); if (cached != null) { - log.log(LogLevel.DEBUG, () -> "Found config " + key + " in cache"); + log.log(Level.FINE, () -> "Found config " + key + " in cache"); return cached; } else { return null; @@ -60,9 +60,6 @@ class MemoryCacheConfigClient implements ConfigSourceClient { } @Override - public void updateSubscribers(RawConfig config) {} - - @Override public DelayedResponses delayedResponses() { return delayedResponses; } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java index 545b962f6ff..4606ec7c072 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ProxyServer.java @@ -5,7 +5,7 @@ import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.log.event.Event; import com.yahoo.vespa.config.RawConfig; @@ -48,7 +48,7 @@ public class ProxyServer implements Runnable { ProxyServer(Spec spec, ConfigSourceSet source, MemoryCache memoryCache, ConfigSourceClient configClient) { this.configSource = source; - log.log(LogLevel.DEBUG, "Using config source '" + source); + log.log(Level.FINE, "Using config source '" + source); this.memoryCache = memoryCache; this.rpcServer = createRpcServer(spec); this.configClient = (configClient == null) ? createRpcClient(rpcServer, source, memoryCache) : configClient; @@ -99,7 +99,7 @@ public class ProxyServer implements Runnable { default: throw new IllegalArgumentException("Cannot set invalid mode '" + modeName + "'"); } - log.log(LogLevel.INFO, "Switched from '" + oldMode.name().toLowerCase() + "' mode to '" + getMode().name().toLowerCase() + "' mode"); + log.log(Level.INFO, "Switched from '" + oldMode.name().toLowerCase() + "' mode to '" + getMode().name().toLowerCase() + "' mode"); } private ConfigProxyRpcServer createRpcServer(Spec spec) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java index 2a33e8c6928..ba58c369afd 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/RpcConfigSourceClient.java @@ -10,22 +10,25 @@ import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigCacheKey; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.concurrent.DelayQueue; -import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.logging.Logger; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; /** @@ -33,7 +36,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; * * @author hmusum */ -class RpcConfigSourceClient implements ConfigSourceClient { +class RpcConfigSourceClient implements ConfigSourceClient, Runnable { private final static Logger log = Logger.getLogger(RpcConfigSourceClient.class.getName()); private static final double timingValuesRatio = 0.8; @@ -42,16 +45,18 @@ class RpcConfigSourceClient implements ConfigSourceClient { private final RpcServer rpcServer; private final ConfigSourceSet configSourceSet; - private final HashMap<ConfigCacheKey, Subscriber> activeSubscribers = new HashMap<>(); - private final Object activeSubscribersLock = new Object(); + private final Map<ConfigCacheKey, Subscriber> activeSubscribers = new ConcurrentHashMap<>(); private final MemoryCache memoryCache; private final DelayedResponses delayedResponses; private final static TimingValues timingValues; - private final ExecutorService exec; + private final ScheduledExecutorService nextConfigScheduler = + Executors.newScheduledThreadPool(1, new DaemonThreadFactory("next config")); + private ScheduledFuture<?> nextConfigFuture; private final JRTConfigRequester requester; // Scheduled executor that periodically checks for requests that have timed out and response should be returned to clients - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1, new DaemonThreadFactory()); - private ScheduledFuture<?> delayedResponseScheduler; + private final ScheduledExecutorService delayedResponsesScheduler = + Executors.newScheduledThreadPool(1, new DaemonThreadFactory("delayed responses")); + private ScheduledFuture<?> delayedResponsesFuture; static { // Proxy should time out before clients upon subscription. @@ -69,14 +74,10 @@ class RpcConfigSourceClient implements ConfigSourceClient { this.memoryCache = memoryCache; this.delayedResponses = new DelayedResponses(); checkConfigSources(); - exec = Executors.newCachedThreadPool(new DaemonThreadFactory("subscriber-")); + nextConfigFuture = nextConfigScheduler.scheduleAtFixedRate(this, 0, 10, MILLISECONDS); requester = JRTConfigRequester.create(configSourceSet, timingValues); - // Wait for 5 seconds initially, then run every second - delayedResponseScheduler = scheduler.scheduleAtFixedRate( - new DelayedResponseHandler(delayedResponses, memoryCache, rpcServer), - 5, - 1, - SECONDS); + DelayedResponseHandler command = new DelayedResponseHandler(delayedResponses, memoryCache, rpcServer); + delayedResponsesFuture = delayedResponsesScheduler.scheduleAtFixedRate(command, 5, 1, SECONDS); } /** @@ -84,7 +85,7 @@ class RpcConfigSourceClient implements ConfigSourceClient { */ private void checkConfigSources() { if (configSourceSet == null || configSourceSet.getSources() == null || configSourceSet.getSources().size() == 0) { - log.log(LogLevel.WARNING, "No config sources defined, could not check connection"); + log.log(Level.WARNING, "No config sources defined, could not check connection"); } else { Request req = new Request("ping"); for (String configSource : configSourceSet.getSources()) { @@ -92,15 +93,15 @@ class RpcConfigSourceClient implements ConfigSourceClient { Target target = supervisor.connect(spec); target.invokeSync(req, 30.0); if (target.isValid()) { - log.log(LogLevel.DEBUG, () -> "Created connection to config source at " + spec.toString()); + log.log(Level.FINE, () -> "Created connection to config source at " + spec.toString()); return; } else { - log.log(LogLevel.INFO, "Could not connect to config source at " + spec.toString()); + log.log(Level.INFO, "Could not connect to config source at " + spec.toString()); } target.close(); } String extra = ""; - log.log(LogLevel.INFO, "Could not connect to any config source in set " + configSourceSet.toString() + + log.log(Level.INFO, "Could not connect to any config source in set " + configSourceSet.toString() + ", please make sure config server(s) are running. " + extra); } } @@ -132,11 +133,11 @@ class RpcConfigSourceClient implements ConfigSourceClient { RawConfig ret = null; if (cachedConfig != null) { - log.log(LogLevel.DEBUG, () -> "Found config " + configCacheKey + " in cache, generation=" + cachedConfig.getGeneration() + + log.log(Level.FINE, () -> "Found config " + configCacheKey + " in cache, generation=" + cachedConfig.getGeneration() + ",configmd5=" + cachedConfig.getConfigMd5()); - log.log(LogLevel.SPAM, () -> "input config=" + input + ",cached config=" + cachedConfig); + log.log(Level.FINEST, () -> "input config=" + input + ",cached config=" + cachedConfig); if (ProxyServer.configOrGenerationHasChanged(cachedConfig, request)) { - log.log(LogLevel.SPAM, () -> "Cached config is not equal to requested, will return it"); + log.log(Level.FINEST, () -> "Cached config is not equal to requested, will return it"); if (delayedResponses.remove(delayedResponse)) { // unless another thread already did it ret = cachedConfig; @@ -153,30 +154,36 @@ class RpcConfigSourceClient implements ConfigSourceClient { } private void subscribeToConfig(RawConfig input, ConfigCacheKey configCacheKey) { - synchronized (activeSubscribersLock) { - if (activeSubscribers.containsKey(configCacheKey)) { - log.log(LogLevel.DEBUG, () -> "Already a subscriber running for: " + configCacheKey); - } else { - log.log(LogLevel.DEBUG, () -> "Could not find good config in cache, creating subscriber for: " + configCacheKey); - UpstreamConfigSubscriber subscriber = - new UpstreamConfigSubscriber(input, this, configSourceSet, timingValues, requester, memoryCache); - try { - subscriber.subscribe(); - activeSubscribers.put(configCacheKey, subscriber); - exec.execute(subscriber); - } catch (ConfigurationRuntimeException e) { - log.log(LogLevel.INFO, "Subscribe for '" + configCacheKey + "' failed, closing subscriber"); - subscriber.cancel(); - } - } + if (activeSubscribers.containsKey(configCacheKey)) return; + + log.log(Level.FINE, () -> "Could not find good config in cache, creating subscriber for: " + configCacheKey); + var subscriber = new Subscriber(input, configSourceSet, timingValues, requester); + try { + subscriber.subscribe(); + activeSubscribers.put(configCacheKey, subscriber); + } catch (ConfigurationRuntimeException e) { + log.log(Level.INFO, "Subscribe for '" + configCacheKey + "' failed, closing subscriber"); + subscriber.cancel(); } } @Override + public void run() { + activeSubscribers.values().forEach(subscriber -> { + if (!subscriber.isClosed()) { + Optional<RawConfig> config = subscriber.nextGeneration(); + config.ifPresent(this::updateWithNewConfig); + } + }); + } + + @Override public void cancel() { shutdownSourceConnections(); - delayedResponseScheduler.cancel(true); - scheduler.shutdown(); + delayedResponsesFuture.cancel(true); + delayedResponsesScheduler.shutdown(); + nextConfigFuture.cancel(true); + nextConfigScheduler.shutdown(); } /** @@ -184,13 +191,9 @@ class RpcConfigSourceClient implements ConfigSourceClient { */ @Override public void shutdownSourceConnections() { - synchronized (activeSubscribersLock) { - for (Subscriber subscriber : activeSubscribers.values()) { - subscriber.cancel(); - } - activeSubscribers.clear(); - } - exec.shutdown(); + activeSubscribers.values().forEach(Subscriber::cancel); + activeSubscribers.clear(); + nextConfigScheduler.shutdown(); requester.close(); } @@ -215,14 +218,14 @@ class RpcConfigSourceClient implements ConfigSourceClient { * @param config new config */ public void updateSubscribers(RawConfig config) { - log.log(LogLevel.DEBUG, () -> "Config updated for " + config.getKey() + "," + config.getGeneration()); + log.log(Level.FINE, () -> "Config updated for " + config.getKey() + "," + config.getGeneration()); DelayQueue<DelayedResponse> responseDelayQueue = delayedResponses.responses(); - log.log(LogLevel.SPAM, () -> "Delayed response queue: " + responseDelayQueue); + log.log(Level.FINEST, () -> "Delayed response queue: " + responseDelayQueue); if (responseDelayQueue.size() == 0) { - log.log(LogLevel.DEBUG, () -> "There exists no matching element on delayed response queue for " + config.getKey()); + log.log(Level.FINE, () -> "There exists no matching element on delayed response queue for " + config.getKey()); return; } else { - log.log(LogLevel.DEBUG, () -> "Delayed response queue has " + responseDelayQueue.size() + " elements"); + log.log(Level.FINE, () -> "Delayed response queue has " + responseDelayQueue.size() + " elements"); } boolean found = false; for (DelayedResponse response : responseDelayQueue.toArray(new DelayedResponse[0])) { @@ -232,17 +235,17 @@ class RpcConfigSourceClient implements ConfigSourceClient { && (config.getGeneration() >= request.getRequestGeneration() || config.getGeneration() == 0)) { if (delayedResponses.remove(response)) { found = true; - log.log(LogLevel.DEBUG, () -> "Call returnOkResponse for " + config.getKey() + "," + config.getGeneration()); + log.log(Level.FINE, () -> "Call returnOkResponse for " + config.getKey() + "," + config.getGeneration()); rpcServer.returnOkResponse(request, config); } else { - log.log(LogLevel.INFO, "Could not remove " + config.getKey() + " from delayedResponses queue, already removed"); + log.log(Level.INFO, "Could not remove " + config.getKey() + " from delayedResponses queue, already removed"); } } } if (!found) { - log.log(LogLevel.DEBUG, () -> "Found no recipient for " + config.getKey() + " in delayed response queue"); + log.log(Level.FINE, () -> "Found no recipient for " + config.getKey() + " in delayed response queue"); } - log.log(LogLevel.DEBUG, () -> "Finished updating config for " + config.getKey() + "," + config.getGeneration()); + log.log(Level.FINE, () -> "Finished updating config for " + config.getKey() + "," + config.getGeneration()); } @Override @@ -250,4 +253,12 @@ class RpcConfigSourceClient implements ConfigSourceClient { return delayedResponses; } + private void updateWithNewConfig(RawConfig newConfig) { + log.log(Level.FINE, () -> "config to be returned for '" + newConfig.getKey() + + "', generation=" + newConfig.getGeneration() + + ", payload=" + newConfig.getPayload()); + memoryCache.update(newConfig); + updateSubscribers(newConfig); + } + } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java index 74c99b7670b..f96d6470679 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/Subscriber.java @@ -1,12 +1,68 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.proxy; +import com.yahoo.config.subscription.ConfigSourceSet; +import com.yahoo.config.subscription.impl.GenericConfigHandle; +import com.yahoo.config.subscription.impl.GenericConfigSubscriber; +import com.yahoo.config.subscription.impl.JRTConfigRequester; + +import java.util.Optional; +import java.util.logging.Level; +import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.config.RawConfig; +import com.yahoo.vespa.config.TimingValues; +import com.yahoo.yolean.Exceptions; + +import java.util.Map; +import java.util.logging.Logger; + /** - * Interface for subscribing to config from upstream config sources. - * * @author hmusum */ -public interface Subscriber extends Runnable { +public class Subscriber { + + private final static Logger log = Logger.getLogger(Subscriber.class.getName()); + + private final RawConfig config; + private final ConfigSourceSet configSourceSet; + private final TimingValues timingValues; + private final GenericConfigSubscriber subscriber; + private GenericConfigHandle handle; + + Subscriber(RawConfig config, ConfigSourceSet configSourceSet, TimingValues timingValues, JRTConfigRequester requester) { + this.config = config; + this.configSourceSet = configSourceSet; + this.timingValues = timingValues; + this.subscriber = new GenericConfigSubscriber(Map.of(configSourceSet, requester)); + } + + void subscribe() { + ConfigKey<?> key = config.getKey(); + handle = subscriber.subscribe(new ConfigKey<>(key.getName(), key.getConfigId(), key.getNamespace()), + config.getDefContent(), configSourceSet, timingValues); + } + + public Optional<RawConfig> nextGeneration() { + if (subscriber.nextGeneration(0)) { + try { + return Optional.of(handle.getRawConfig()); + } catch (Exception e) { // To avoid thread throwing exception and loop never running this again + log.log(Level.WARNING, "Got exception: " + Exceptions.toMessageString(e)); + } catch (Throwable e) { + com.yahoo.protect.Process.logAndDie("Got error, exiting: " + Exceptions.toMessageString(e)); + } + } + return Optional.empty(); + } + + public void cancel() { + if (subscriber != null) { + subscriber.close(); + } + } + + boolean isClosed() { + return subscriber.isClosed(); + } - void cancel(); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java deleted file mode 100644 index d8a8c5ce941..00000000000 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/UpstreamConfigSubscriber.java +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.config.proxy; - -import com.yahoo.config.subscription.ConfigSourceSet; -import com.yahoo.config.subscription.impl.GenericConfigHandle; -import com.yahoo.config.subscription.impl.GenericConfigSubscriber; -import com.yahoo.config.subscription.impl.JRTConfigRequester; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.RawConfig; -import com.yahoo.vespa.config.TimingValues; -import com.yahoo.yolean.Exceptions; - -import java.util.Map; -import java.util.logging.Logger; - -/** - * @author hmusum - */ -public class UpstreamConfigSubscriber implements Subscriber { - - private final static Logger log = Logger.getLogger(UpstreamConfigSubscriber.class.getName()); - - private final RawConfig config; - private final ConfigSourceClient configSourceClient; - private final ConfigSourceSet configSourceSet; - private final TimingValues timingValues; - private final JRTConfigRequester requester; - private final MemoryCache memoryCache; - private GenericConfigSubscriber subscriber; - private GenericConfigHandle handle; - - UpstreamConfigSubscriber(RawConfig config, ConfigSourceClient configSourceClient, ConfigSourceSet configSourceSet, - TimingValues timingValues, JRTConfigRequester requester, - MemoryCache memoryCache) { - this.config = config; - this.configSourceClient = configSourceClient; - this.configSourceSet = configSourceSet; - this.timingValues = timingValues; - this.requester = requester; - this.memoryCache = memoryCache; - } - - void subscribe() { - subscriber = new GenericConfigSubscriber(Map.of(configSourceSet, requester)); - ConfigKey<?> key = config.getKey(); - handle = subscriber.subscribe(new ConfigKey<>(key.getName(), key.getConfigId(), key.getNamespace()), - config.getDefContent(), configSourceSet, timingValues); - } - - @Override - public void run() { - do { - if (! subscriber.nextGeneration()) continue; - - try { - updateWithNewConfig(handle); - } catch (Exception e) { // To avoid thread throwing exception and loop never running this again - log.log(LogLevel.WARNING, "Got exception: " + Exceptions.toMessageString(e)); - } catch (Throwable e) { - com.yahoo.protect.Process.logAndDie("Got error, exiting: " + Exceptions.toMessageString(e)); - } - } while (!subscriber.isClosed()); - } - - private void updateWithNewConfig(GenericConfigHandle handle) { - RawConfig newConfig = handle.getRawConfig(); - log.log(LogLevel.DEBUG, () -> "config to be returned for '" + newConfig.getKey() + - "', generation=" + newConfig.getGeneration() + - ", payload=" + newConfig.getPayload()); - memoryCache.update(newConfig); - configSourceClient.updateSubscribers(newConfig); - } - - @Override - public void cancel() { - if (subscriber != null) { - subscriber.close(); - } - } - -} diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java index eec045cdb0a..247ef8a4dbf 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/CachedFilesMaintainer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.proxy.filedistribution; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.filedistribution.FileDownloader; import java.io.File; @@ -63,18 +63,18 @@ class CachedFilesMaintainer implements Runnable { File[] files = directory.listFiles(); if (files != null) filesOnDisk.addAll(Arrays.stream(files).map(File::getName).collect(Collectors.toSet())); - log.log(LogLevel.DEBUG, "Files on disk (in " + directory + "): " + filesOnDisk); + log.log(Level.FINE, "Files on disk (in " + directory + "): " + filesOnDisk); Set<String> filesToDelete = filesOnDisk .stream() .filter(fileReference -> isFileLastModifiedBefore(new File(directory, fileReference), deleteNotUsedSinceInstant)) .collect(Collectors.toSet()); if (filesToDelete.size() > 0) { - log.log(LogLevel.INFO, "Files that can be deleted in " + directory + " (not used since " + deleteNotUsedSinceInstant + "): " + filesToDelete); + log.log(Level.INFO, "Files that can be deleted in " + directory + " (not used since " + deleteNotUsedSinceInstant + "): " + filesToDelete); filesToDelete.forEach(fileReference -> { File file = new File(directory, fileReference); if (!IOUtils.recursiveDeleteDir(file)) - log.log(LogLevel.WARNING, "Could not delete " + file.getAbsolutePath()); + log.log(Level.WARNING, "Could not delete " + file.getAbsolutePath()); }); } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java index 7db9761d86a..a25e86926a1 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java @@ -10,7 +10,7 @@ import com.yahoo.jrt.Request; import com.yahoo.jrt.StringArray; import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReferenceDownload; @@ -106,7 +106,7 @@ class FileDistributionRpcServer { } private void setFileReferencesToDownload(Request req) { - log.log(LogLevel.DEBUG, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(Level.FINE, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); Arrays.stream(req.parameters().get(0).asStringArray()) .map(FileReference::new) .forEach(fileReference -> downloader.downloadIfNeeded(new FileReferenceDownload(fileReference))); @@ -115,14 +115,14 @@ class FileDistributionRpcServer { private void downloadFile(Request req) { FileReference fileReference = new FileReference(req.parameters().get(0).asString()); - log.log(LogLevel.DEBUG, () -> "getFile() called for file reference '" + fileReference.value() + "'"); + log.log(Level.FINE, () -> "getFile() called for file reference '" + fileReference.value() + "'"); Optional<File> file = downloader.getFile(fileReference); if (file.isPresent()) { new RequestTracker().trackRequest(file.get().getParentFile()); req.returnValues().add(new StringValue(file.get().getAbsolutePath())); - log.log(LogLevel.DEBUG, () -> "File reference '" + fileReference.value() + "' available at " + file.get()); + log.log(Level.FINE, () -> "File reference '" + fileReference.value() + "' available at " + file.get()); } else { - log.log(LogLevel.INFO, "File reference '" + fileReference.value() + "' not found, returning error"); + log.log(Level.INFO, "File reference '" + fileReference.value() + "' not found, returning error"); req.setError(fileReferenceDoesNotExists, "File reference '" + fileReference.value() + "' not found"); } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java index 47f478ea4d7..206c0359850 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/RequestTracker.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.proxy.filedistribution; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.File; import java.time.Instant; @@ -21,10 +21,10 @@ class RequestTracker { void trackRequest(File file) { String absolutePath = file.getAbsolutePath(); if ( ! file.exists()) - log.log(LogLevel.WARNING, "Could not find file '" + absolutePath + "'"); + log.log(Level.WARNING, "Could not find file '" + absolutePath + "'"); if ( ! file.setLastModified(Instant.now().toEpochMilli())) - log.log(LogLevel.WARNING, "Could not set last modified timestamp for '" + absolutePath + "'"); + log.log(Level.WARNING, "Could not set last modified timestamp for '" + absolutePath + "'"); } } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java index 592f5211eed..d51144a5715 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/UrlDownloadRpcServer.java @@ -6,7 +6,7 @@ import com.yahoo.jrt.Method; import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.defaults.Defaults; import net.jpountz.xxhash.XXHashFactory; @@ -79,20 +79,20 @@ class UrlDownloadRpcServer { setIfModifiedSince(connection, downloadDir); // don't download if we already have the file if (connection.getResponseCode() == 200) { - log.log(LogLevel.INFO, "Downloading URL '" + url + "'"); + log.log(Level.INFO, "Downloading URL '" + url + "'"); downloadFile(req, connection, downloadDir); } else if (connection.getResponseCode() == 304) { - log.log(LogLevel.INFO, "URL '" + url + "' already downloaded (server response: 304)"); + log.log(Level.INFO, "URL '" + url + "' already downloaded (server response: 304)"); req.returnValues().add(new StringValue(new File(downloadDir, CONTENTS_FILE_NAME).getAbsolutePath())); } else { - log.log(LogLevel.ERROR, "Download of URL '" + url + "' got server response: " + connection.getResponseCode()); + log.log(Level.SEVERE, "Download of URL '" + url + "' got server response: " + connection.getResponseCode()); req.setError(HTTP_ERROR, String.valueOf(connection.getResponseCode())); } } catch (Throwable e) { - log.log(LogLevel.ERROR, "Download of URL '" + url + "' got exception: " + e.getMessage()); + log.log(Level.SEVERE, "Download of URL '" + url + "' got exception: " + e.getMessage()); req.setError(INTERNAL_ERROR, "Download of URL '" + url + "' internal error: " + e.getMessage()); } req.returnRequest(); @@ -111,11 +111,11 @@ class UrlDownloadRpcServer { writeLastModifiedTimestamp(downloadDir, connection.getLastModified()); new RequestTracker().trackRequest(downloadDir); req.returnValues().add(new StringValue(contentsPath.getAbsolutePath())); - log.log(LogLevel.DEBUG, () -> "URL '" + url + "' available at " + contentsPath); - log.log(LogLevel.INFO, String.format("Download of URL '%s' done in %.3f seconds", + log.log(Level.FINE, () -> "URL '" + url + "' available at " + contentsPath); + log.log(Level.INFO, String.format("Download of URL '%s' done in %.3f seconds", url, (System.currentTimeMillis() -start) / 1000.0)); } else { - log.log(LogLevel.ERROR, "Downloaded URL '" + url + "' not found, returning error"); + log.log(Level.SEVERE, "Downloaded URL '" + url + "' not found, returning error"); req.setError(DOES_NOT_EXIST, "Downloaded '" + url + "' not found"); } } diff --git a/config-proxy/src/main/sh/vespa-config-ctl.sh b/config-proxy/src/main/sh/vespa-config-ctl.sh index 91e4cd39bd2..bf75b497fea 100755 --- a/config-proxy/src/main/sh/vespa-config-ctl.sh +++ b/config-proxy/src/main/sh/vespa-config-ctl.sh @@ -130,7 +130,7 @@ case $1 in echo "Waiting for config proxy to start" fail=true - for ((sleepcount=0;$sleepcount<600;sleepcount=$sleepcount+1)) ; do + for ((sleepcount=0;$sleepcount<1800;sleepcount=$sleepcount+1)) ; do sleep 0.1 if [ -f $P_CONFIG_PROXY ] && kill -0 `cat $P_CONFIG_PROXY` && vespa-ping-configproxy -s $hname 2>/dev/null then @@ -153,11 +153,6 @@ case $1 in elif ! vespa-ping-configproxy -s $hname then echo "failed to ping config proxy $hname" 1>&2 - # TODO: Dump stack trace for debugging, remove after April 2020 - kill -3 `pgrep -f -n configproxy` - sleep 2 - kill -3 `pgrep -f -n configproxy` - sleep 2 kill -3 `pgrep -f -n configproxy` fi diff --git a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java index 06e55eef4fa..2126d673da0 100644 --- a/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java +++ b/config-proxy/src/test/java/com/yahoo/vespa/config/proxy/MockConfigSourceClient.java @@ -54,9 +54,6 @@ public class MockConfigSourceClient implements ConfigSourceClient{ } @Override - public void updateSubscribers(RawConfig config) { } - - @Override public DelayedResponses delayedResponses() { return delayedResponses; } } diff --git a/config/pom.xml b/config/pom.xml index ad036c442d1..6e4e26ed0f1 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -53,7 +53,7 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> + <artifactId>defaults</artifactId> <version>${project.version}</version> </dependency> <dependency> diff --git a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java index 474c9f7a4db..74d4d378e5d 100644 --- a/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java +++ b/config/src/main/java/com/yahoo/config/subscription/CfgConfigPayloadBuilder.java @@ -4,13 +4,15 @@ package com.yahoo.config.subscription; import com.yahoo.collections.Pair; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.StringNode; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ConfigPayloadBuilder; import java.util.ArrayList; import java.util.List; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.FINEST; + /** * Deserializes config payload (cfg format) to a ConfigPayload. * @@ -37,7 +39,7 @@ public class CfgConfigPayloadBuilder { parseLine(line, lineNum, payloadBuilder); lineNum++; } - log.log(LogLevel.SPAM, () -> "payload=" + payloadBuilder.toString()); + log.log(FINEST, () -> "payload=" + payloadBuilder.toString()); return payloadBuilder; } @@ -48,7 +50,7 @@ public class CfgConfigPayloadBuilder { String field = fieldAndValue.getFirst(); String value = fieldAndValue.getSecond(); if (field==null || value==null) { - log.log(LogLevel.DEBUG, () -> "Got field without value in line " + lineNum + ": " + line + ", skipping"); + log.log(FINE, () -> "Got field without value in line " + lineNum + ": " + line + ", skipping"); return; } field=field.trim(); diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java index 3020f49b32e..d279fc42251 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigDebug.java @@ -2,16 +2,17 @@ package com.yahoo.config.subscription; import com.yahoo.config.ConfigInstance; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import java.util.logging.Logger; +import static java.util.logging.Level.INFO; + // Debug class that provides useful helper routines public class ConfigDebug { public static void logDebug(Logger logger, long timestamp, ConfigKey<?> key, String logmessage) { if (key.getConfigId().matches(".*container.?\\d+.*") || key.getConfigId().matches(".*doc.api.*")) { - logger.log(LogLevel.INFO, timestamp + " " + key + " " + logmessage); + logger.log(INFO, timestamp + " " + key + " " + logmessage); } } diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java index 7472439d6a4..e0605dce30d 100755 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSourceSet.java @@ -1,8 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription; -import com.yahoo.log.LogLevel; - import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; @@ -10,6 +8,7 @@ import java.util.List; import java.util.Set; import java.util.logging.Logger; +import static java.util.logging.Level.INFO; /** * An immutable set of connection endpoints, where each endpoint points to either a @@ -98,7 +97,7 @@ public class ConfigSourceSet implements ConfigSource { public static ConfigSourceSet createDefault() { String configSources = System.getenv("VESPA_CONFIG_SOURCES"); if (configSources != null) { - log.log(LogLevel.INFO, "Using config sources from VESPA_CONFIG_SOURCES: " + configSources); + log.log(INFO, "Using config sources from VESPA_CONFIG_SOURCES: " + configSources); return new ConfigSourceSet(checkSourcesSyntax(configSources)); } else { String[] def = {"tcp/localhost:" + System.getProperty("vespa.config.port", "19090")}; diff --git a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java index 3891d710fa3..814222989a1 100644 --- a/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java +++ b/config/src/main/java/com/yahoo/config/subscription/ConfigSubscriber.java @@ -5,7 +5,6 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.impl.ConfigSubscription; import com.yahoo.config.subscription.impl.JRTConfigRequester; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.TimingValues; import com.yahoo.yolean.Exceptions; @@ -14,8 +13,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; + import java.util.logging.Logger; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.SEVERE; +import static java.util.logging.Level.WARNING; import static java.util.stream.Collectors.toList; /** @@ -326,7 +329,7 @@ public class ConfigSubscriber implements AutoCloseable { h.subscription().close(); } closeRequesters(); - log.log(LogLevel.DEBUG, () -> "Config subscriber has been closed."); + log.log(FINE, () -> "Config subscriber has been closed."); } /** @@ -434,7 +437,7 @@ public class ConfigSubscriber implements AutoCloseable { if (handle.isChanged()) singleSubscriber.configure(handle.getConfig()); } } catch (Exception e) { - log.log(LogLevel.ERROR, "Exception from config system, continuing config thread: " + Exceptions.toMessageString(e)); + log.log(SEVERE, "Exception from config system, continuing config thread: " + Exceptions.toMessageString(e)); } } } @@ -477,7 +480,7 @@ public class ConfigSubscriber implements AutoCloseable { protected void finalize() throws Throwable { try { if (!isClosed()) { - log.log(LogLevel.WARNING, stackTraceAtConstruction, + log.log(WARNING, stackTraceAtConstruction, () -> String.format("%s: Closing subscription from finalizer() - close() has not been called (keys=%s)", super.toString(), subscriptionHandles.stream().map(handle -> handle.subscription().getKey().toString()).collect(toList()))); diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java index 60c920d1dfa..365f6a4d4c5 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/FileConfigSubscription.java @@ -1,10 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription.impl; -import java.io.File; -import java.io.IOException; -import java.util.Arrays; - import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.CfgConfigPayloadBuilder; @@ -13,7 +9,12 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.io.IOUtils; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.log.LogLevel; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; + +import static java.util.logging.Level.FINE; /** * Subscription used when config id is file:... @@ -37,12 +38,12 @@ public class FileConfigSubscription<T extends ConfigInstance> extends ConfigSubs public boolean nextConfig(long timeout) { if (!file.exists() && !file.isFile()) throw new IllegalArgumentException("Not a file: "+file); if (checkReloaded()) { - log.log(LogLevel.DEBUG, "User forced config reload at " + System.currentTimeMillis()); + log.log(FINE, "User forced config reload at " + System.currentTimeMillis()); // User forced reload setConfigIfChanged(updateConfig()); ConfigState<T> configState = getConfigState(); - log.log(LogLevel.DEBUG, "Config updated at " + System.currentTimeMillis() + ", changed: " + configState.isConfigChanged()); - log.log(LogLevel.DEBUG, "Config: " + configState.getConfig().toString()); + log.log(FINE, "Config updated at " + System.currentTimeMillis() + ", changed: " + configState.isConfigChanged()); + log.log(FINE, "Config: " + configState.getConfig().toString()); return true; } if (file.lastModified()!=ts) { diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java index 8ce3449fba5..eec18b93e71 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/GenericJRTConfigSubscription.java @@ -5,13 +5,14 @@ import java.util.List; import com.yahoo.config.subscription.ConfigSource; import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; +import static java.util.logging.Level.FINE; + /** * A JRT subscription which does not use the config class, but {@link com.yahoo.vespa.config.RawConfig} instead. * Used by config proxy. @@ -33,7 +34,7 @@ public class GenericJRTConfigSubscription extends JRTConfigSubscription<RawConfi @Override protected void setNewConfig(JRTClientConfigRequest jrtReq) { setConfig(jrtReq.getNewGeneration(), jrtReq.responseIsInternalRedeploy(), RawConfig.createFromResponseParameters(jrtReq) ); - log.log(LogLevel.DEBUG, () -> "in setNewConfig, config=" + this.getConfigState().getConfig()); + log.log(FINE, () -> "in setNewConfig, config=" + this.getConfigState().getConfig()); } // This method is overridden because config needs to have its generation diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index 49c5dcd343c..51266f21846 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -6,7 +6,6 @@ import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Request; import com.yahoo.jrt.RequestWaiter; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.ErrorCode; @@ -24,6 +23,12 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.FINEST; +import static java.util.logging.Level.INFO; +import static java.util.logging.Level.SEVERE; +import static java.util.logging.Level.WARNING; + /** * This class fetches config payload using JRT, and acts as the callback target. * It uses the {@link JRTConfigSubscription} and {@link JRTClientConfigRequest} @@ -90,9 +95,9 @@ public class JRTConfigRequester implements RequestWaiter { if ( ! req.validateParameters()) throw new ConfigurationRuntimeException("Error in parameters for config request: " + req); double jrtClientTimeout = getClientTimeout(req); - log.log(LogLevel.DEBUG, () -> "Requesting config for " + sub + " on connection " + connection + log.log(FINE, () -> "Requesting config for " + sub + " on connection " + connection + " with client timeout " + jrtClientTimeout + - (log.isLoggable(LogLevel.SPAM) ? (",defcontent=" + req.getDefContent().asString()) : "")); + (log.isLoggable(FINEST) ? (",defcontent=" + req.getDefContent().asString()) : "")); connection.invokeAsync(req.getRequest(), jrtClientTimeout, this); } @@ -110,7 +115,7 @@ public class JRTConfigRequester implements RequestWaiter { sub.setException(e); } else { // Very unlikely - log.log(Level.SEVERE, "Failed to get subscription object from JRT config callback: " + + log.log(SEVERE, "Failed to get subscription object from JRT config callback: " + Exceptions.toMessageString(e)); } } @@ -118,11 +123,11 @@ public class JRTConfigRequester implements RequestWaiter { private void doHandle(JRTConfigSubscription<ConfigInstance> sub, JRTClientConfigRequest jrtReq, Connection connection) { boolean validResponse = jrtReq.validateResponse(); - log.log(LogLevel.DEBUG, () -> "Request callback " + (validResponse ? "valid" : "invalid") + ". Req: " + jrtReq + "\nSpec: " + connection); + log.log(FINE, () -> "Request callback " + (validResponse ? "valid" : "invalid") + ". Req: " + jrtReq + "\nSpec: " + connection); if (sub.getState() == ConfigSubscription.State.CLOSED) return; // Avoid error messages etc. after closing Trace trace = jrtReq.getResponseTrace(); trace.trace(TRACELEVEL, "JRTConfigRequester.doHandle()"); - log.log(LogLevel.SPAM, () -> trace.toString()); + log.log(FINEST, () -> trace.toString()); if (validResponse) { handleOKRequest(jrtReq, sub, connection); } else { @@ -134,20 +139,20 @@ public class JRTConfigRequester implements RequestWaiter { private void logWhenErrorResponse(JRTClientConfigRequest jrtReq, Connection connection) { switch (jrtReq.errorCode()) { case com.yahoo.jrt.ErrorCode.CONNECTION: - log.log(LogLevel.DEBUG, () -> "Request callback failed: " + jrtReq.errorMessage() + + log.log(FINE, () -> "Request callback failed: " + jrtReq.errorMessage() + "\nConnection spec: " + connection); break; case ErrorCode.APPLICATION_NOT_LOADED: case ErrorCode.UNKNOWN_VESPA_VERSION: if (noApplicationWarningLogged.isBefore(Instant.now().minus(delayBetweenWarnings))) { - log.log(LogLevel.WARNING, "Request callback failed: " + ErrorCode.getName(jrtReq.errorCode()) + + log.log(WARNING, "Request callback failed: " + ErrorCode.getName(jrtReq.errorCode()) + ". Connection spec: " + connection.getAddress() + ", error message: " + jrtReq.errorMessage()); noApplicationWarningLogged = Instant.now(); } break; default: - log.log(LogLevel.WARNING, "Request callback failed. Req: " + jrtReq + "\nSpec: " + connection.getAddress() + + log.log(WARNING, "Request callback failed. Req: " + jrtReq + "\nSpec: " + connection.getAddress() + " . Req error message: " + jrtReq.errorMessage()); break; } @@ -157,7 +162,7 @@ public class JRTConfigRequester implements RequestWaiter { final boolean configured = (sub.getConfigState().getConfig() != null); if (configured) { // The subscription object has an "old" config, which is all we have to offer back now - log.log(LogLevel.INFO, "Failure of config subscription, clients will keep existing config until resolved: " + sub); + log.log(INFO, "Failure of config subscription, clients will keep existing config until resolved: " + sub); } ErrorType errorType = ErrorType.getErrorType(jrtReq.errorCode()); connectionPool.setError(connection, jrtReq.errorCode()); @@ -192,7 +197,7 @@ public class JRTConfigRequester implements RequestWaiter { Connection connection) { transientFailures++; if (suspendWarningLogged.isBefore(Instant.now().minus(delayBetweenWarnings))) { - log.log(LogLevel.INFO, "Connection to " + connection.getAddress() + + log.log(INFO, "Connection to " + connection.getAddress() + " failed or timed out, clients will keep existing config, will keep trying."); suspendWarningLogged = Instant.now(); } @@ -218,7 +223,7 @@ public class JRTConfigRequester implements RequestWaiter { if (sub.getState() != ConfigSubscription.State.OPEN) return; fatalFailures++; // The logging depends on whether we are configured or not. - Level logLevel = sub.getConfigState().getConfig() == null ? LogLevel.DEBUG : LogLevel.INFO; + Level logLevel = sub.getConfigState().getConfig() == null ? Level.FINE : Level.INFO; String logMessage = "Request for config " + jrtReq.getShortDescription() + "' failed with error code " + jrtReq.errorCode() + " (" + jrtReq.errorMessage() + "), scheduling new connect " + " in " + delay + " ms"; @@ -236,7 +241,7 @@ public class JRTConfigRequester implements RequestWaiter { noApplicationWarningLogged = Instant.MIN; connection.setSuccess(); sub.setLastCallBackOKTS(Instant.now()); - log.log(LogLevel.DEBUG, () -> "OK response received in handleOkRequest: " + jrtReq); + log.log(FINE, () -> "OK response received in handleOkRequest: " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { // We only want this latest generation to be in the queue, we do not preserve history in this system sub.getReqQueue().clear(); @@ -260,8 +265,8 @@ public class JRTConfigRequester implements RequestWaiter { private void scheduleNextRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription<?> sub, long delay, long timeout) { long delayBeforeSendingRequest = (delay < 0) ? 0 : delay; JRTClientConfigRequest jrtReqNew = jrtReq.nextRequest(timeout); - log.log(LogLevel.SPAM, () -> timingValues.toString()); - log.log(LogLevel.DEBUG, () -> "Scheduling new request " + delayBeforeSendingRequest + " millis from now for " + jrtReqNew.getConfigKey()); + log.log(FINEST, () -> timingValues.toString()); + log.log(FINE, () -> "Scheduling new request " + delayBeforeSendingRequest + " millis from now for " + jrtReqNew.getConfigKey()); scheduler.schedule(new GetConfigTask(jrtReqNew, sub), delayBeforeSendingRequest, TimeUnit.MILLISECONDS); } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java index a94a135f9d8..44f6d65ee65 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigSubscription.java @@ -1,25 +1,26 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.subscription.impl; -import java.time.Duration; -import java.time.Instant; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - import com.yahoo.config.ConfigInstance; import com.yahoo.config.subscription.ConfigInterruptedException; import com.yahoo.config.subscription.ConfigSource; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.config.subscription.ConfigSubscriber; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; -import com.yahoo.vespa.config.JRTConnectionPool; import com.yahoo.vespa.config.TimingValues; import com.yahoo.vespa.config.protocol.CompressionType; import com.yahoo.vespa.config.protocol.JRTClientConfigRequest; import com.yahoo.vespa.config.protocol.Payload; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.INFO; + /** * A JRT config subscription uses one {@link JRTConfigRequester} to fetch config using Vespa RPC from a config source, typically proxy or server * @@ -90,7 +91,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc // timed out, we know nothing new. return false; } - log.log(LogLevel.DEBUG, () -> "Polled queue and found config " + jrtReq); + log.log(FINE, () -> "Polled queue and found config " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { setInternalRedeploy(jrtReq.responseIsInternalRedeploy()); if (jrtReq.hasUpdatedConfig()) { @@ -192,7 +193,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc @Override public void reload(long generation) { - log.log(LogLevel.DEBUG, "reload() is without effect on a JRTConfigSubscription."); + log.log(FINE, "reload() is without effect on a JRTConfigSubscription."); } void setLastCallBackOKTS(Instant lastCallBackOKTS) { @@ -204,7 +205,7 @@ public class JRTConfigSubscription<T extends ConfigInstance> extends ConfigSubsc static void printStatus(JRTClientConfigRequest request, String message) { final String name = request.getConfigKey().getName(); if (name.equals("components") || name.equals("chains")) { - log.log(LogLevel.INFO, message + ":" + name + ":" + ", request=" + request); + log.log(INFO, message + ":" + name + ":" + ", request=" + request); } } } diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 134352736b6..b4df42c802e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -5,13 +5,12 @@ import com.yahoo.config.ConfigBuilder; import com.yahoo.config.ConfigInstance; import com.yahoo.config.FileReference; import com.yahoo.config.UrlReference; -import com.yahoo.log.LogLevel; -import com.yahoo.yolean.Exceptions; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; import com.yahoo.slime.Type; import com.yahoo.text.Utf8; +import com.yahoo.yolean.Exceptions; import java.io.File; import java.lang.reflect.Constructor; @@ -27,6 +26,10 @@ import java.util.Set; import java.util.Stack; import java.util.logging.Logger; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.FINEST; +import static java.util.logging.Level.INFO; + /** * A utility class that can be used to apply a payload to a config builder. * @@ -224,7 +227,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { } catch (InvocationTargetException | IllegalAccessException e) { throw new RuntimeException("Name: " + methodName + ", value '" + value + "'", e); } catch (NoSuchMethodException e) { - log.log(LogLevel.INFO, "Skipping unknown field " + methodName + " in " + rootBuilder); + log.log(INFO, "Skipping unknown field " + methodName + " in " + rootBuilder); } } @@ -279,7 +282,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { } catch (InvocationTargetException | IllegalAccessException e) { throw new RuntimeException("Name: " + methodName + ", value '" + value + "'", e); } catch (NoSuchMethodException e) { - log.log(LogLevel.INFO, "Skipping unknown field " + methodName + " in " + builder.getClass()); + log.log(INFO, "Skipping unknown field " + methodName + " in " + builder.getClass()); } } @@ -479,11 +482,11 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { } private void debug(String message) { - log.log(LogLevel.DEBUG, () -> message); + log.log(FINE, () -> message); } private void trace(String message) { - log.log(LogLevel.SPAM, () -> message); + log.log(FINEST, () -> message); } private void printStack() { diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java index 326c1287468..2c1d00f295e 100644 --- a/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnectionPool.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.config; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -13,6 +12,8 @@ import java.util.Map; import java.util.concurrent.ThreadLocalRandom; import java.util.logging.Logger; +import static java.util.logging.Level.FINE; + /** * A pool of JRT connections to a config source (either a config server or a config proxy). * The current connection is chosen randomly when calling {#link #setNewCurrentConnection} @@ -74,7 +75,7 @@ public class JRTConnectionPool implements ConnectionPool { public synchronized JRTConnection setNewCurrentConnection() { List<JRTConnection> sources = getSources(); currentConnection = sources.get(ThreadLocalRandom.current().nextInt(0, sources.size())); - log.log(LogLevel.DEBUG, () -> "Choosing new connection: " + currentConnection); + log.log(FINE, () -> "Choosing new connection: " + currentConnection); return currentConnection; } diff --git a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java index 863e95c5625..2356b400343 100644 --- a/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java +++ b/config/src/main/java/com/yahoo/vespa/config/UrlDownloader.java @@ -8,12 +8,14 @@ import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.defaults.Defaults; import java.io.File; import java.util.logging.Logger; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.INFO; + /** * @author lesters */ @@ -47,7 +49,7 @@ public class UrlDownloader { Request request = new Request("frt.rpc.ping"); target.invokeSync(request, 5.0); if (! request.isError()) { - log.log(LogLevel.DEBUG, () -> "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); + log.log(FINE, () -> "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); return; } else { target.close(); @@ -78,7 +80,7 @@ public class UrlDownloader { request.parameters().add(new StringValue(urlReference.value())); double rpcTimeout = Math.min(timeLeft, 60 * 60.0); - log.log(LogLevel.DEBUG, () -> "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); + log.log(FINE, () -> "InvokeSync waitFor " + urlReference + " with " + rpcTimeout + " seconds timeout"); target.invokeSync(request, rpcTimeout); if (request.checkReturnTypes("s")) { @@ -86,7 +88,7 @@ public class UrlDownloader { } else if (!request.isError()) { throw new RuntimeException("Invalid response: " + request.returnValues()); } else if (temporaryError(request)) { - log.log(LogLevel.INFO, "Retrying waitFor for " + urlReference + ": " + request.errorCode() + " -- " + request.errorMessage()); + log.log(INFO, "Retrying waitFor for " + urlReference + ": " + request.errorCode() + " -- " + request.errorMessage()); try { Thread.sleep(1000); } catch (InterruptedException e) { diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java index 8c7c7f2703e..478dc6c45d6 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/RequestValidation.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.protocol; -import com.yahoo.log.LogLevel; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ErrorCode; @@ -10,6 +9,8 @@ import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static java.util.logging.Level.INFO; + /** * Static utility methods for verifying common request properties. * @@ -23,31 +24,31 @@ public class RequestValidation { public static int validateRequest(JRTConfigRequest request) { ConfigKey<?> key = request.getConfigKey(); if (!RequestValidation.verifyName(key.getName())) { - log.log(LogLevel.INFO, "Illegal name '" + key.getName() + "'"); + log.log(INFO, "Illegal name '" + key.getName() + "'"); return ErrorCode.ILLEGAL_NAME; } if (!RequestValidation.verifyNamespace(key.getNamespace())) { - log.log(LogLevel.INFO, "Illegal name space '" + key.getNamespace() + "'"); + log.log(INFO, "Illegal name space '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_NAME_SPACE; } if (!RequestValidation.verifyMd5(key.getMd5())) { - log.log(LogLevel.INFO, "Illegal md5 sum '" + key.getNamespace() + "'"); + log.log(INFO, "Illegal md5 sum '" + key.getNamespace() + "'"); return ErrorCode.ILLEGAL_DEF_MD5; } if (!RequestValidation.verifyMd5(request.getRequestConfigMd5())) { - log.log(LogLevel.INFO, "Illegal config md5 '" + request.getRequestConfigMd5() + "'"); + log.log(INFO, "Illegal config md5 '" + request.getRequestConfigMd5() + "'"); return ErrorCode.ILLEGAL_CONFIG_MD5; } if (!RequestValidation.verifyGeneration(request.getRequestGeneration())) { - log.log(LogLevel.INFO, "Illegal generation '" + request.getRequestGeneration() + "'"); + log.log(INFO, "Illegal generation '" + request.getRequestGeneration() + "'"); return ErrorCode.ILLEGAL_GENERATION; } if (!RequestValidation.verifyTimeout(request.getTimeout())) { - log.log(LogLevel.INFO, "Illegal timeout '" + request.getTimeout() + "'"); + log.log(INFO, "Illegal timeout '" + request.getTimeout() + "'"); return ErrorCode.ILLEGAL_TIMEOUT; } if (!RequestValidation.verifyHostname(request.getClientHostName())) { - log.log(LogLevel.INFO, "Illegal client host name '" + request.getClientHostName() + "'"); + log.log(INFO, "Illegal client host name '" + request.getClientHostName() + "'"); return ErrorCode.ILLEGAL_CLIENT_HOSTNAME; } return 0; diff --git a/configdefinitions/src/vespa/lb-services.def b/configdefinitions/src/vespa/lb-services.def index cc496a99c20..33c568061fe 100644 --- a/configdefinitions/src/vespa/lb-services.def +++ b/configdefinitions/src/vespa/lb-services.def @@ -4,9 +4,6 @@ namespace=cloud.config -# Enable proxy-protocol for nginx upstreams -nginxUpstreamProxyProtocol bool default=false - # Active rotation given as flag 'active' for a prod region in deployment.xml # Default true for now (since code in config-model to set it is not ready yet), should have no default value tenants{}.applications{}.activeRotation bool default=true diff --git a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java index d34b1fd949f..c034ea79447 100644 --- a/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java +++ b/configserver-flags/src/main/java/com/yahoo/vespa/configserver/flags/http/FlagsHandler.java @@ -5,7 +5,7 @@ import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.Path; import com.yahoo.vespa.configserver.flags.FlagsDb; @@ -48,7 +48,7 @@ public class FlagsHandler extends LoggingRequestHandler { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(LogLevel.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 95c8733b540..f6846a7448e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -24,7 +24,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.transaction.NestedTransaction; @@ -215,7 +215,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye try (ActionTimer timer = timerFor(applicationId, "deployment.prepareMillis")) { ConfigChangeActions actions = session.prepare(logger, prepareParams, currentActiveApplicationSet, tenant.getPath(), now); logConfigChangeActions(actions, logger); - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " prepared successfully. "); return new PrepareResult(sessionId, actions, deployLog); } } @@ -380,17 +380,17 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye remoteSession = getRemoteSession(tenant, sessionId); Transaction deleteTransaction = remoteSession.createDeleteTransaction(); deleteTransaction.commit(); - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Waiting for session " + sessionId + " to be deleted"); if ( ! waitTime.isZero() && localSessionHasBeenDeleted(applicationId, sessionId, waitTime)) { - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Session " + sessionId + " deleted"); } else { deleteTransaction.rollbackOrLog(); throw new InternalServerException(applicationId + " was not deleted (waited " + waitTime + "), session " + sessionId); } } catch (NotFoundException e) { // For the case where waiting timed out in a previous attempt at deleting the application, continue and do the steps below - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Active session exists, but has not been deleted properly. Trying to cleanup"); + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Active session exists, but has not been deleted properly. Trying to cleanup"); } NestedTransaction transaction = new NestedTransaction(); @@ -400,7 +400,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye transaction.add(tenantApplications.createDeleteTransaction(applicationId)); hostProvisioner.ifPresent(provisioner -> provisioner.remove(transaction, applicationId)); - transaction.onCommitted(() -> log.log(LogLevel.INFO, "Deleted " + applicationId)); + transaction.onCommitted(() -> log.log(Level.INFO, "Deleted " + applicationId)); transaction.commit(); return true; } @@ -447,17 +447,17 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye .map(FileReference::value) .collect(Collectors.toSet())); } catch (Exception e) { - log.log(LogLevel.WARNING, "Getting file references in use for '" + application + "' failed", e); + log.log(Level.WARNING, "Getting file references in use for '" + application + "' failed", e); } } - log.log(LogLevel.DEBUG, "File references in use : " + fileReferencesInUse); + log.log(Level.FINE, "File references in use : " + fileReferencesInUse); // Find those on disk that are not in use Set<String> fileReferencesOnDisk = new HashSet<>(); File[] filesOnDisk = fileReferencesPath.listFiles(); if (filesOnDisk != null) fileReferencesOnDisk.addAll(Arrays.stream(filesOnDisk).map(File::getName).collect(Collectors.toSet())); - log.log(LogLevel.DEBUG, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); + log.log(Level.FINE, "File references on disk (in " + fileReferencesPath + "): " + fileReferencesOnDisk); Instant instant = Instant.now().minus(keepFileReferences); Set<String> fileReferencesToDelete = fileReferencesOnDisk @@ -466,11 +466,11 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye .filter(fileReference -> isFileLastModifiedBefore(new File(fileReferencesPath, fileReference), instant)) .collect(Collectors.toSet()); if (fileReferencesToDelete.size() > 0) { - log.log(LogLevel.INFO, "Will delete file references not in use: " + fileReferencesToDelete); + log.log(Level.INFO, "Will delete file references not in use: " + fileReferencesToDelete); fileReferencesToDelete.forEach(fileReference -> { File file = new File(fileReferencesPath, fileReference); if ( ! IOUtils.recursiveDeleteDir(file)) - log.log(LogLevel.WARNING, "Could not delete " + file.getAbsolutePath()); + log.log(Level.WARNING, "Could not delete " + file.getAbsolutePath()); }); } return fileReferencesToDelete; @@ -498,10 +498,10 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye if (session == null) throw new NotFoundException("Remote session " + sessionId + " not found"); return session.ensureApplicationLoaded().getForVersionOrLatest(version, clock.instant()); } catch (NotFoundException e) { - log.log(LogLevel.WARNING, "Failed getting application for '" + applicationId + "': " + e.getMessage()); + log.log(Level.WARNING, "Failed getting application for '" + applicationId + "': " + e.getMessage()); throw e; } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed getting application for '" + applicationId + "'", e); + log.log(Level.WARNING, "Failed getting application for '" + applicationId + "'", e); throw e; } } @@ -783,9 +783,9 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } private void cleanupTempDirectory(File tempDir) { - logger.log(LogLevel.DEBUG, "Deleting tmp dir '" + tempDir + "'"); + logger.log(Level.FINE, "Deleting tmp dir '" + tempDir + "'"); if (!IOUtils.recursiveDeleteDir(tempDir)) { - logger.log(LogLevel.WARNING, "Not able to delete tmp dir '" + tempDir + "'"); + logger.log(Level.WARNING, "Not able to delete tmp dir '" + tempDir + "'"); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index 2a426e4cccc..d628702758b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.TransientException; import com.yahoo.container.handler.VipStatus; import com.yahoo.container.jdisc.state.StateMonitor; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.rpc.RpcServer; import com.yahoo.vespa.config.server.version.VersionState; import com.yahoo.yolean.Exceptions; @@ -97,7 +97,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable this.sleepTimeWhenRedeployingFails = Duration.ofSeconds(configserverConfig.sleepTimeWhenRedeployingFails()); this.exitIfRedeployingApplicationsFails = exitIfRedeployingApplicationsFails; rpcServerExecutor = Executors.newSingleThreadExecutor(new DaemonThreadFactory("config server RPC server")); - log.log(LogLevel.DEBUG, "Bootstrap mode: " + mode + ", VIP status mode: " + vipStatusMode); + log.log(Level.FINE, "Bootstrap mode: " + mode + ", VIP status mode: " + vipStatusMode); initializing(vipStatusMode); switch (mode) { case BOOTSTRAP_IN_SEPARATE_THREAD: @@ -118,16 +118,16 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable @Override public void deconstruct() { - log.log(LogLevel.INFO, "Stopping config server"); + log.log(Level.INFO, "Stopping config server"); down(); server.stop(); - log.log(LogLevel.DEBUG, "RPC server stopped"); + log.log(Level.FINE, "RPC server stopped"); rpcServerExecutor.shutdown(); serverThread.ifPresent(thread -> { try { thread.join(); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, "Error joining server thread on shutdown: " + e.getMessage()); + log.log(Level.WARNING, "Error joining server thread on shutdown: " + e.getMessage()); } }); } @@ -139,7 +139,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable try { Thread.sleep(1000); } catch (InterruptedException e) { - log.log(LogLevel.ERROR, "Got interrupted", e); + log.log(Level.SEVERE, "Got interrupted", e); break; } } while (server.isRunning()); @@ -148,7 +148,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable public void start() { if (versionState.isUpgraded()) { - log.log(LogLevel.INFO, "Config server upgrading from " + versionState.storedVersion() + " to " + log.log(Level.INFO, "Config server upgrading from " + versionState.storedVersion() + " to " + versionState.currentVersion() + ". Redeploying all applications"); try { if ( ! redeployAllApplications()) { @@ -156,9 +156,9 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable return; // Status will not be set to 'up' since we return here } versionState.saveNewVersion(); - log.log(LogLevel.INFO, "All applications redeployed successfully"); + log.log(Level.INFO, "All applications redeployed successfully"); } catch (Exception e) { - log.log(LogLevel.ERROR, "Redeployment of applications failed", e); + log.log(Level.SEVERE, "Redeployment of applications failed", e); redeployingApplicationsFailed(); return; // Status will not be set to 'up' since we return here } @@ -193,7 +193,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable try { Thread.sleep(10); } catch (InterruptedException e) { - log.log(LogLevel.ERROR, "Got interrupted", e); + log.log(Level.SEVERE, "Got interrupted", e); break; } } @@ -211,14 +211,14 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable do { applicationsNotRedeployed = redeployApplications(applicationsNotRedeployed); if ( ! applicationsNotRedeployed.isEmpty()) { - log.log(LogLevel.INFO, "Redeployment of " + applicationsNotRedeployed + + log.log(Level.INFO, "Redeployment of " + applicationsNotRedeployed + " failed, will retry in " + sleepTimeWhenRedeployingFails); Thread.sleep(sleepTimeWhenRedeployingFails.toMillis()); } } while ( ! applicationsNotRedeployed.isEmpty() && Instant.now().isBefore(end)); if ( ! applicationsNotRedeployed.isEmpty()) { - log.log(LogLevel.ERROR, "Redeploying applications not finished after " + maxDurationOfRedeployment + + log.log(Level.SEVERE, "Redeploying applications not finished after " + maxDurationOfRedeployment + ", exiting, applications that failed redeployment: " + applicationsNotRedeployed); return false; } @@ -243,10 +243,10 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable } catch (ExecutionException e) { ApplicationId app = f.getKey(); if (e.getCause() instanceof TransientException) { - log.log(LogLevel.INFO, "Redeploying " + app + + log.log(Level.INFO, "Redeploying " + app + " failed with transient error, will retry after bootstrap: " + Exceptions.toMessageString(e)); } else { - log.log(LogLevel.WARNING, "Redeploying " + app + " failed, will retry", e); + log.log(Level.WARNING, "Redeploying " + app + " failed, will retry", e); failedDeployments.add(app); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java index 3e9783cabe9..c788b73720d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerDB.java @@ -5,7 +5,7 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.ConfigDefinitionDir; import com.yahoo.config.model.application.provider.Bundle; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.defaults.Defaults; import java.io.File; @@ -35,7 +35,7 @@ public class ConfigServerDB { try { initialize(configserverConfig.configModelPluginDir()); } catch (IllegalArgumentException e) { - log.log(LogLevel.ERROR, "Error initializing serverdb: " + e.getMessage()); + log.log(Level.SEVERE, "Error initializing serverdb: " + e.getMessage()); } catch (IOException e) { throw new RuntimeException("Unable to initialize server db", e); } @@ -68,10 +68,10 @@ public class ConfigServerDB { for (String pluginDirectory : pluginDirectories) { bundles.addAll(Bundle.getBundles(new File(pluginDirectory))); } - log.log(LogLevel.DEBUG, "Found " + bundles.size() + " bundles"); + log.log(Level.FINE, "Found " + bundles.size() + " bundles"); List<Bundle> addedBundles = new ArrayList<>(); for (Bundle bundle : bundles) { - log.log(LogLevel.DEBUG, "Bundle in " + bundle.getFile().getAbsolutePath() + " appears to contain " + bundle.getDefEntries().size() + " entries"); + log.log(Level.FINE, "Bundle in " + bundle.getFile().getAbsolutePath() + " appears to contain " + bundle.getDefEntries().size() + " entries"); configDefinitionDir.addConfigDefinitionsFromBundle(bundle, addedBundles); addedBundles.add(bundle); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java index d46c551fa7e..db10d9dcc73 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -11,7 +11,7 @@ import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.Zone; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.GenerationCounter; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.model.SuperModelConfigProvider; @@ -127,7 +127,7 @@ public class SuperModelManager implements SuperModelProvider { public void markAsComplete() { // Invoked on component graph bootstrap (even before ConfigServerBootstrap), // there is no need to bump generation counter. - logger.log(LogLevel.INFO, "Super model is complete"); + logger.log(Level.INFO, "Super model is complete"); SuperModel newSuperModel = getSuperModel().cloneAsComplete(); superModelConfigProvider = new SuperModelConfigProvider(newSuperModel, zone, flagSource); listeners.forEach(listener -> listener.notifyOfCompleteness(newSuperModel)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java index 51c5be1b1f0..6fcfde80510 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelRequestHandler.java @@ -7,7 +7,7 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.ConfigDefinitionRepo; import com.yahoo.component.Version; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.protocol.ConfigResponse; @@ -74,10 +74,10 @@ public class SuperModelRequestHandler implements RequestHandler { @Override public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { - log.log(LogLevel.DEBUG, () -> "SuperModelRequestHandler resolving " + req + " for app id '" + appId + "'"); + log.log(Level.FINE, () -> "SuperModelRequestHandler resolving " + req + " for app id '" + appId + "'"); if (handler != null) { ConfigResponse configResponse = handler.resolveConfig(req); - log.log(LogLevel.DEBUG, () -> "SuperModelRequestHandler returning response for config " + req + + log.log(Level.FINE, () -> "SuperModelRequestHandler returning response for config " + req + " with generation " + configResponse.getGeneration()); return configResponse; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java index 30c68b3830c..15a9b8f74d0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/UserConfigDefinitionRepo.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server; import com.google.common.base.Splitter; import com.yahoo.config.model.api.ConfigDefinitionRepo; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.buildergen.ConfigDefinition; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; @@ -32,7 +32,7 @@ public class UserConfigDefinitionRepo implements ConfigDefinitionRepo { defs.put(dKey, new ConfigDefinition(dKey.getName(), Splitter.on("\n").splitToList(payload).toArray(new String[0]))); } } else { - log.log(LogLevel.WARNING, "Path " + appPath + " does not exist, not able to load add user config definitions"); + log.log(Level.WARNING, "Path " + appPath + " does not exist, not able to load add user config definitions"); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index c8f3bcd5b3f..2560badbf43 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -6,7 +6,7 @@ import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.Model; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigCacheKey; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigKey; @@ -149,11 +149,11 @@ public class Application implements ModelResult { } private boolean logDebug() { - return log.isLoggable(LogLevel.DEBUG); + return log.isLoggable(Level.FINE); } private void debug(String message) { - log.log(LogLevel.DEBUG, TenantRepository.logPre(getId())+message); + log.log(Level.FINE, TenantRepository.logPre(getId())+message); } private ConfigDefinition getTargetDef(GetConfigRequest req) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java index c94f739b958..eeb4d31fe12 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStream.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.application; import com.google.common.io.ByteStreams; import com.google.common.io.Files; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.http.BadRequestException; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.v2.ApplicationApiHandler; @@ -84,23 +84,23 @@ public class CompressedApplicationInputStream implements AutoCloseable { } private void decompressInto(File application) throws IOException { - log.log(LogLevel.DEBUG, "Application is in " + application.getAbsolutePath()); + log.log(Level.FINE, "Application is in " + application.getAbsolutePath()); int entries = 0; ArchiveEntry entry; while ((entry = ais.getNextEntry()) != null) { - log.log(LogLevel.DEBUG, "Unpacking " + entry.getName()); + log.log(Level.FINE, "Unpacking " + entry.getName()); File outFile = new File(application, entry.getName()); // FIXME/TODO: write more tests that break this logic. I have a feeling it is not very robust. if (entry.isDirectory()) { if (!(outFile.exists() && outFile.isDirectory())) { - log.log(LogLevel.DEBUG, "Creating dir: " + outFile.getAbsolutePath()); + log.log(Level.FINE, "Creating dir: " + outFile.getAbsolutePath()); boolean res = outFile.mkdirs(); if (!res) { - log.log(LogLevel.WARNING, "Could not create dir " + entry.getName()); + log.log(Level.WARNING, "Could not create dir " + entry.getName()); } } } else { - log.log(LogLevel.DEBUG, "Creating output file: " + outFile.getAbsolutePath()); + log.log(Level.FINE, "Creating output file: " + outFile.getAbsolutePath()); // Create parent dir if necessary String parent = outFile.getParent(); @@ -113,7 +113,7 @@ public class CompressedApplicationInputStream implements AutoCloseable { entries++; } if (entries == 0) { - log.log(LogLevel.WARNING, "Not able to read any entries from " + application.getName()); + log.log(Level.WARNING, "Not able to read any entries from " + application.getName()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index e841dd9b8d9..824cc932ecc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -8,7 +8,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.http.JSONResponse; import org.glassfish.jersey.client.ClientProperties; @@ -72,7 +72,7 @@ public class ConfigConvergenceChecker extends AbstractComponent { /** Check all services in given application. Returns the minimum current generation of all services */ public ServiceListResponse servicesToCheck(Application application, URI requestUrl, Duration timeoutPerService) { - log.log(LogLevel.DEBUG, () -> "Finding services to check config convergence for in '" + application); + log.log(Level.FINE, () -> "Finding services to check config convergence for in '" + application); List<ServiceInfo> servicesToCheck = new ArrayList<>(); application.getModel().getHosts() .forEach(host -> host.getServices().stream() diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java index f37225a7e7b..092db354ffd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/FileDistributionStatus.java @@ -10,7 +10,7 @@ import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.vespa.config.server.http.JSONResponse; @@ -56,7 +56,7 @@ public class FileDistributionStatus extends AbstractComponent { try { hostStatuses.add(future.get()); } catch (InterruptedException | ExecutionException e) { - log.log(LogLevel.WARNING, "Failed getting file distribution status", e); + log.log(Level.WARNING, "Failed getting file distribution status", e); } }); return createStatusForAllHosts(hostStatuses); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java index c1d29329594..8c53a9bc9d5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/HttpProxy.java @@ -6,7 +6,7 @@ import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.http.HttpFetcher; import com.yahoo.vespa.config.server.http.NotFoundException; @@ -58,7 +58,7 @@ public class HttpProxy { try { url = new URL(urlString); } catch (MalformedURLException e) { - logger.log(LogLevel.WARNING, "Badly formed url: " + urlString, e); + logger.log(Level.WARNING, "Badly formed url: " + urlString, e); return HttpErrorResponse.internalServerError("Failed to construct URL for backend"); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java index b64aa1c771c..ac994e8405d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/PermanentApplicationPackage.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.cloud.config.ConfigserverConfig; import static com.yahoo.vespa.defaults.Defaults.getDefaults; @@ -25,7 +25,7 @@ public class PermanentApplicationPackage { File app = new File(getDefaults().underVespaHome(config.applicationDirectory())); applicationPackage = Optional.ofNullable(app.exists() ? FilesApplicationPackage.fromFile(app) : null); if (applicationPackage.isPresent()) { - log.log(LogLevel.DEBUG, "Detected permanent application package in '" + + log.log(Level.FINE, "Detected permanent application package in '" + getDefaults().underVespaHome(config.applicationDirectory()) + "'. This might add extra services to config models"); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 7ff5d41485d..4f04724a0a8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.application; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.transaction.Transaction; @@ -171,11 +171,11 @@ public class TenantApplications { private void applicationRemoved(ApplicationId applicationId) { reloadHandler.removeApplication(applicationId); - log.log(LogLevel.INFO, TenantRepository.logPre(applicationId) + "Application removed: " + applicationId); + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Application removed: " + applicationId); } private void applicationAdded(ApplicationId applicationId) { - log.log(LogLevel.DEBUG, TenantRepository.logPre(applicationId) + "Application added: " + applicationId); + log.log(Level.FINE, TenantRepository.logPre(applicationId) + "Application added: " + applicationId); } private Path applicationPath(ApplicationId id) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java index eaaa4f93bd1..c33c5ff9f57 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLogger.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; + import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; @@ -33,7 +34,7 @@ public class DeployHandlerLogger implements DeployLogger { @Override public void log(Level level, String message) { - if ((level == LogLevel.FINE || + if ((level == Level.FINE || level == LogLevel.DEBUG || level == LogLevel.SPAM) && !verbose) { @@ -45,7 +46,7 @@ public class DeployHandlerLogger implements DeployLogger { entry.setString("level", level.getName()); entry.setString("message", fullMsg); // Also tee to a normal log, Vespa log for example, but use level fine - log.log(LogLevel.FINE, fullMsg); + log.log(Level.FINE, fullMsg); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 079a05167b0..af4e85109e6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.Provisioner; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.ActivationConflictException; @@ -150,7 +150,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { session.waitUntilActivated(timeoutBudget); - log.log(LogLevel.INFO, session.logPre() + "Session " + session.getSessionId() + + log.log(Level.INFO, session.logPre() + "Session " + session.getSessionId() + " activated successfully using " + (hostProvisioner.isPresent() ? hostProvisioner.get().getClass().getSimpleName() : "no host provisioner") + ". Config generation " + session.getMetaData().getGeneration() + @@ -197,12 +197,12 @@ public class Deployment implements com.yahoo.config.provision.Deployment { private void checkIfActiveHasChanged(LocalSession session, LocalSession currentActiveSession, boolean ignoreStaleSessionFailure) { long activeSessionAtCreate = session.getActiveSessionAtCreate(); - log.log(LogLevel.DEBUG, currentActiveSession.logPre() + "active session id at create time=" + activeSessionAtCreate); + log.log(Level.FINE, currentActiveSession.logPre() + "active session id at create time=" + activeSessionAtCreate); if (activeSessionAtCreate == 0) return; // No active session at create long sessionId = session.getSessionId(); long currentActiveSessionSessionId = currentActiveSession.getSessionId(); - log.log(LogLevel.DEBUG, currentActiveSession.logPre() + "sessionId=" + sessionId + + log.log(Level.FINE, currentActiveSession.logPre() + "sessionId=" + sessionId + ", current active session=" + currentActiveSessionSessionId); if (currentActiveSession.isNewerThan(activeSessionAtCreate) && currentActiveSessionSessionId != sessionId) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 89714392869..cd06f7208e4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -154,6 +154,8 @@ public class ModelContextImpl implements ModelContext { private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final double defaultTermwiseLimit; private final double defaultSoftStartSeconds; + private final double threadPoolSizeFactor; + private final double queueSizefactor; private final Optional<AthenzDomain> athenzDomain; public Properties(ApplicationId applicationId, @@ -190,6 +192,10 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); defaultTopKprobability = Flags.DEFAULT_TOP_K_PROBABILITY.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + threadPoolSizeFactor = Flags.DEFAULT_THREADPOOL_SIZE_FACTOR.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + queueSizefactor = Flags.DEFAULT_QUEUE_SIZE_FACTOR.bindTo(flagSource) + .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.athenzDomain = athenzDomain; } @@ -242,8 +248,18 @@ public class ModelContextImpl implements ModelContext { @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } + @Override + public double threadPoolSizeFactor() { + return threadPoolSizeFactor; + } + + @Override + public double queueSizeFactor() { + return queueSizefactor; + } + public double defaultSoftStartSeconds() { - return 0; + return defaultSoftStartSeconds; } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 09f5178cf6b..e2cf84d6715 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -12,7 +12,7 @@ import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.serialization.AllocatedHostsSerializer; import com.yahoo.io.reader.NamedReader; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; @@ -67,7 +67,7 @@ public class ZooKeeperClient { createZooKeeperNodes(); break; } catch (RuntimeException e) { - logger.log(LogLevel.FINE, "ZK init failed, retrying: " + e); + logger.log(Level.FINE, "ZK init failed, retrying: " + e); retries--; if (retries == 0) { throw e; @@ -262,7 +262,7 @@ public class ZooKeeperClient { writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.USER_DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); writeConfigDefinition(key.getName(), key.getNamespace(), getZooKeeperAppPath(ConfigCurator.DEFCONFIGS_ZK_SUBPATH).getAbsolute(), contents); } - logger.log(LogLevel.FINE, configDefs.size() + " user config definitions"); + logger.log(Level.FINE, configDefs.size() + " user config definitions"); } private void writeConfigDefinition(String name, String namespace, String path, String data) { @@ -298,7 +298,7 @@ public class ZooKeeperClient { configCurator.deleteRecurse(getZooKeeperAppPath(null).append(subPath).getAbsolute()); } } catch (Exception e) { - logger.log(LogLevel.WARNING, "Could not clean up in zookeeper"); + logger.log(Level.WARNING, "Could not clean up in zookeeper"); //Might be called in an exception handler before re-throw, so do not throw here. } } @@ -320,7 +320,7 @@ public class ZooKeeperClient { private void logFine(String msg) { if (logFine) { - logger.log(LogLevel.FINE, msg); + logger.log(Level.FINE, msg); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index f710eec27ba..9f65c0e3c6e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -5,7 +5,7 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.model.api.FileDistribution; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import net.jpountz.xxhash.XXHash64; import net.jpountz.xxhash.XXHashFactory; @@ -32,7 +32,7 @@ public class FileDirectory { try { ensureRootExist(); } catch (IllegalArgumentException e) { - log.log(LogLevel.WARNING, "Failed creating directory in constructor, will retry on demand : " + e.toString()); + log.log(Level.WARNING, "Failed creating directory in constructor, will retry on demand : " + e.toString()); } } @@ -82,10 +82,10 @@ public class FileDirectory { if (file.isDirectory()) { return Files.walk(file.toPath(), 100).map(path -> { try { - log.log(LogLevel.DEBUG, "Calculating hash for '" + path + "'"); + log.log(Level.FINE, "Calculating hash for '" + path + "'"); return hash(path.toFile(), hasher); } catch (IOException e) { - log.log(LogLevel.WARNING, "Failed getting hash from '" + path + "'"); + log.log(Level.WARNING, "Failed getting hash from '" + path + "'"); return 0; } }).mapToLong(Number::longValue).sum(); @@ -119,7 +119,7 @@ public class FileDirectory { File existingFile = destinationDir.toPath().resolve(source.getName()).toFile(); if ( ! existingFile.exists() || ! computeHash(existingFile).equals(hashOfFileToBeAdded)) { - log.log(LogLevel.ERROR, "Directory for file reference '" + fileReference.value() + + log.log(Level.SEVERE, "Directory for file reference '" + fileReference.value() + "' has content that does not match its hash, deleting everything in " + destinationDir.getAbsolutePath()); IOUtils.recursiveDeleteDir(destinationDir); @@ -143,17 +143,17 @@ public class FileDirectory { File destination = new File(tempDestinationDir.toFile(), source.getName()); if (!destinationDir.exists()) { destinationDir.mkdir(); - log.log(LogLevel.DEBUG, "file reference ' " + reference.value() + "', source: " + source.getAbsolutePath() ); + log.log(Level.FINE, "file reference ' " + reference.value() + "', source: " + source.getAbsolutePath() ); if (source.isDirectory()) { - log.log(LogLevel.DEBUG, "Copying source " + source.getAbsolutePath() + " to " + destination.getAbsolutePath()); + log.log(Level.FINE, "Copying source " + source.getAbsolutePath() + " to " + destination.getAbsolutePath()); IOUtils.copyDirectory(source, destination, -1); } else { copyFile(source, destination); } if (!destinationDir.exists()) { - log.log(LogLevel.DEBUG, "Moving from " + tempDestinationDir + " to " + destinationDir.getAbsolutePath()); + log.log(Level.FINE, "Moving from " + tempDestinationDir + " to " + destinationDir.getAbsolutePath()); if ( ! tempDestinationDir.toFile().renameTo(destinationDir)) { - log.log(LogLevel.WARNING, "Failed moving '" + tempDestinationDir.toFile().getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'."); + log.log(Level.WARNING, "Failed moving '" + tempDestinationDir.toFile().getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'."); } } else { IOUtils.copyDirectory(tempDestinationDir.toFile(), destinationDir, -1); @@ -168,7 +168,7 @@ public class FileDirectory { private void logfileInfo(File file ) throws IOException { BasicFileAttributes basicFileAttributes = Files.readAttributes(file.toPath(), BasicFileAttributes.class); - log.log(LogLevel.DEBUG, "Adding file " + file.getAbsolutePath() + " (created " + basicFileAttributes.creationTime() + + log.log(Level.FINE, "Adding file " + file.getAbsolutePath() + " (created " + basicFileAttributes.creationTime() + ", modified " + basicFileAttributes.lastModifiedTime() + ", size " + basicFileAttributes.size() + ")"); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java index 2178af6915d..56bdeab76b2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionImpl.java @@ -10,7 +10,7 @@ import com.yahoo.jrt.Spec; import com.yahoo.jrt.StringArray; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.defaults.Defaults; import java.io.File; @@ -49,7 +49,7 @@ public class FileDistributionImpl implements FileDistribution, RequestWaiter { Request request = new Request("filedistribution.setFileReferencesToDownload"); request.setContext(target); request.parameters().add(new StringArray(fileReferences.stream().map(FileReference::value).toArray(String[]::new))); - log.log(LogLevel.DEBUG, "Executing " + request.methodName() + " against " + target); + log.log(Level.FINE, "Executing " + request.methodName() + " against " + target); target.invokeAsync(request, rpcTimeout, this); } @@ -58,7 +58,7 @@ public class FileDistributionImpl implements FileDistribution, RequestWaiter { public void handleRequestDone(Request req) { Target target = (Target) req.getContext(); if (req.isError()) { - log.log(LogLevel.DEBUG, req.methodName() + " failed for " + target + ": " + req.errorCode() + " (" + req.errorMessage() + ")"); + log.log(Level.FINE, req.methodName() + " failed for " + target + ": " + req.errorCode() + " (" + req.errorMessage() + ")"); } if (target != null) target.close(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index cb4945bdab3..805ee2bef95 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -10,7 +10,7 @@ import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.net.HostName; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; @@ -101,7 +101,7 @@ public class FileServer { try { return root.getFile(reference).exists(); } catch (IllegalArgumentException e) { - log.log(LogLevel.DEBUG, "Failed locating file reference '" + reference + "' with error " + e.toString()); + log.log(Level.FINE, "Failed locating file reference '" + reference + "' with error " + e.toString()); } return false; } @@ -117,7 +117,7 @@ public class FileServer { private void serveFile(FileReference reference, Receiver target) { File file = root.getFile(reference); - log.log(LogLevel.DEBUG, () -> "Start serving reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); + log.log(Level.FINE, () -> "Start serving reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); boolean success = false; String errorDescription = "OK"; FileReferenceData fileData = FileReferenceDataBlob.empty(reference, file.getName()); @@ -131,9 +131,9 @@ public class FileServer { try { target.receive(fileData, new ReplayStatus(success ? 0 : 1, success ? "OK" : errorDescription)); - log.log(LogLevel.DEBUG, "Done serving file reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); + log.log(Level.FINE, "Done serving file reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed serving file reference '" + reference.value() + "': " + Exceptions.toMessageString(e)); + log.log(Level.WARNING, "Failed serving file reference '" + reference.value() + "': " + Exceptions.toMessageString(e)); } finally { fileData.close(); } @@ -156,7 +156,7 @@ public class FileServer { } private void serveFileInternal(String fileReference, boolean downloadFromOtherSourceIfNotFound, Request request, Receiver receiver) { - log.log(LogLevel.DEBUG, () -> "Received request for reference '" + fileReference + "' from " + request.target()); + log.log(Level.FINE, () -> "Received request for reference '" + fileReference + "' from " + request.target()); boolean fileExists; try { @@ -178,10 +178,10 @@ public class FileServer { // This is to avoid config servers asking each other for a file that does not exist private boolean download(String fileReference, boolean downloadFromOtherSourceIfNotFound) { if (downloadFromOtherSourceIfNotFound) { - log.log(LogLevel.DEBUG, "File not found, downloading from another source"); + log.log(Level.FINE, "File not found, downloading from another source"); return download(fileReference).isPresent(); } else { - log.log(LogLevel.DEBUG, "File not found, will not download from another source since request came from another config server"); + log.log(Level.FINE, "File not found, will not download from another source since request came from another config server"); return false; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java index a7f8b8164a5..ec37f2598e0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java @@ -7,7 +7,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import com.google.common.collect.Collections2; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; /** * A host registry with a mapping between hosts (hostname as a String) and some type T @@ -28,7 +28,7 @@ public class HostRegistry<T> implements HostValidator<T> { public synchronized void update(T key, Collection<String> newHosts) { verifyHosts(key, newHosts); Collection<String> currentHosts = getHostsForKey(key); - log.log(LogLevel.DEBUG, () -> "Setting hosts for key '" + key + "', " + + log.log(Level.FINE, () -> "Setting hosts for key '" + key + "', " + "newHosts: " + newHosts + ", " + "currentHosts: " + currentHosts); Collection<String> removedHosts = getRemovedHosts(newHosts, currentHosts); @@ -71,14 +71,14 @@ public class HostRegistry<T> implements HostValidator<T> { private void removeHosts(Collection<String> removedHosts) { for (String host : removedHosts) { - log.log(LogLevel.DEBUG, () -> "Removing " + host); + log.log(Level.FINE, () -> "Removing " + host); host2KeyMap.remove(host); } } private void addHosts(T key, Collection<String> newHosts) { for (String host : newHosts) { - log.log(LogLevel.DEBUG, () -> "Adding " + host); + log.log(Level.FINE, () -> "Adding " + host); host2KeyMap.put(host, key); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java index 6559292645c..7dc6b5185a6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; @@ -33,7 +33,7 @@ public class HttpErrorResponse extends HttpResponse { root.setString("error-code", errorType); root.setString("message", msg); if (code != 200) { - log.log(LogLevel.INFO, "Returning response with response code " + code + ", error-code:" + errorType + ", message=" + msg); + log.log(Level.INFO, "Returning response with response code " + code + ", error-code:" + errorType + ", message=" + msg); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java index 0397678fd41..7458c82942f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandler.java @@ -5,7 +5,7 @@ import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.tenant.TenantRepository; @@ -41,7 +41,7 @@ public class HttpGetConfigHandler extends HttpHandler { } private ConfigResponse resolveConfig(HttpConfigRequest request) { - log.log(LogLevel.DEBUG, "nocache=" + request.noCache()); + log.log(Level.FINE, "nocache=" + request.noCache()); ConfigResponse config = requestHandler.resolveConfig(ApplicationId.defaultId(), request, Optional.empty()); if (config == null) HttpConfigRequest.throwModelNotReady(); return config; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index 7cdb8d3a60c..e92eb52028b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.config.provision.OutOfCapacityException; import com.yahoo.vespa.config.server.ActivationConflictException; import com.yahoo.yolean.Exceptions; @@ -32,7 +32,7 @@ public class HttpHandler extends LoggingRequestHandler { @Override public HttpResponse handle(HttpRequest request) { - log.log(LogLevel.DEBUG, request.getMethod() + " " + request.getUri().toString()); + log.log(Level.FINE, request.getMethod() + " " + request.getUri().toString()); try { switch (request.getMethod()) { case POST: @@ -71,7 +71,7 @@ public class HttpHandler extends LoggingRequestHandler { } catch (LoadBalancerServiceException e) { return HttpErrorResponse.loadBalancerNotReady(getMessage(e, request)); } catch (Exception e) { - log.log(LogLevel.WARNING, "Unexpected exception handling a config server request", e); + log.log(Level.WARNING, "Unexpected exception handling a config server request", e); return HttpErrorResponse.internalServerError(getMessage(e, request)); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java index d69c0084180..075b4bc329b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SessionContentStatusListResponse.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; @@ -30,7 +30,7 @@ class SessionContentStatusListResponse extends SessionResponse { element.setString("status", f.getMetaData().getStatus()); element.setString("md5", f.getMetaData().getMd5()); element.setString("name", urlBase + f.getPath()); - log.log(LogLevel.DEBUG, "Adding file " + urlBase + f.getPath()); + log.log(Level.FINE, "Adding file " + urlBase + f.getPath()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java index 76a022df580..9c77442dcc8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/SimpleHttpFetcher.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.http; import ai.vespa.util.http.VespaHttpClientBuilder; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.apache.http.HttpEntity; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; @@ -42,15 +42,15 @@ public class SimpleHttpFetcher implements HttpFetcher { } } catch (ConnectTimeoutException | SocketTimeoutException e) { String message = "Timed out after " + params.readTimeoutMs + " ms reading response from " + url; - logger.log(LogLevel.WARNING, message, e); + logger.log(Level.WARNING, message, e); throw new RequestTimeoutException(message); } catch (IOException e) { String message = "Failed to get response from " + url; - logger.log(LogLevel.WARNING, message, e); + logger.log(Level.WARNING, message, e); throw new InternalServerException(message); } catch (URISyntaxException e) { String message = "Invalid URL: " + e.getMessage(); - logger.log(LogLevel.WARNING, message, e); + logger.log(Level.WARNING, message, e); throw new InternalServerException(message, e); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java index 2d5cce2d4f1..514e0c1c8ae 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/TesterClient.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.http; import ai.vespa.util.http.VespaHttpClientBuilder; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.yolean.Exceptions; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; @@ -59,7 +59,7 @@ public class TesterClient { } private HttpResponse execute(HttpUriRequest request, String messageIfRequestFails) { - logger.log(LogLevel.DEBUG, "Sending request to tester container " + request.getURI().toString()); + logger.log(Level.FINE, "Sending request to tester container " + request.getURI().toString()); try { return new ProxyResponse(httpClient.execute(request)); } catch (IOException e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java index d8385456866..ffe3d39b524 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HostHandler.java @@ -9,7 +9,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.jdisc.application.BindingMatch; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.host.HostRegistry; @@ -39,22 +39,22 @@ public class HostHandler extends HttpHandler { @Override public HttpResponse handleGET(HttpRequest request) { String hostname = getBindingMatch(request).group(2); - log.log(LogLevel.DEBUG, "hostname=" + hostname); + log.log(Level.FINE, "hostname=" + hostname); HostRegistry<TenantName> tenantHostRegistry = hostRegistries.getTenantHostRegistry(); - log.log(LogLevel.DEBUG, "hosts in tenant host registry '" + tenantHostRegistry + "' " + tenantHostRegistry.getAllHosts()); + log.log(Level.FINE, "hosts in tenant host registry '" + tenantHostRegistry + "' " + tenantHostRegistry.getAllHosts()); TenantName tenant = tenantHostRegistry.getKeyForHost(hostname); if (tenant == null) return createError(hostname); - log.log(LogLevel.DEBUG, "tenant=" + tenant); + log.log(Level.FINE, "tenant=" + tenant); HostRegistry<ApplicationId> applicationIdHostRegistry = hostRegistries.getApplicationHostRegistry(tenant); ApplicationId applicationId; if (applicationIdHostRegistry == null) return createError(hostname); applicationId = applicationIdHostRegistry.getKeyForHost(hostname); - log.log(LogLevel.DEBUG, "applicationId=" + applicationId); + log.log(Level.FINE, "applicationId=" + applicationId); if (applicationId == null) { return createError(hostname); } else { - log.log(LogLevel.DEBUG, "hosts in application host registry '" + applicationIdHostRegistry + "' " + applicationIdHostRegistry.getAllHosts()); + log.log(Level.FINE, "hosts in application host registry '" + applicationIdHostRegistry + "' " + applicationIdHostRegistry.getAllHosts()); return new HostResponse(Response.Status.OK, applicationId, zone); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java index 49c0cb7427f..b4ce833cb75 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandler.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.http.v2; import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.RequestHandler; import com.yahoo.vespa.config.server.tenant.TenantRepository; @@ -41,7 +41,7 @@ public class HttpGetConfigHandler extends HttpHandler { } private ConfigResponse resolveConfig(HttpConfigRequest request, RequestHandler requestHandler) { - log.log(LogLevel.DEBUG, "nocache=" + request.noCache()); + log.log(Level.FINE, "nocache=" + request.noCache()); ConfigResponse config = requestHandler.resolveConfig(request.getApplicationId(), request, Optional.empty()); if (config == null) HttpConfigRequest.throwModelNotReady(); return config; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java index dc2c23dc9a3..4cf59135026 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandler.java @@ -12,7 +12,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; @@ -55,7 +55,7 @@ public class SessionActiveHandler extends SessionHandler { // Overridden to make sure we are logging when this low-level handling of timeout happens @Override public void handleTimeout(Request request, ResponseHandler responseHandler) { - log.log(LogLevel.ERROR, "activate timed out for " + request.getUri(), new RuntimeException("activate timed out")); + log.log(Level.SEVERE, "activate timed out for " + request.getUri(), new RuntimeException("activate timed out")); super.handleTimeout(request, responseHandler); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java index 4f64b73b403..42d3ace6bd9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/metrics/ClusterMetricsRetriever.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.server.metrics; import ai.vespa.util.http.VespaHttpClientBuilder; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; @@ -76,7 +76,7 @@ public class ClusterMetricsRetriever { throw new RuntimeException(e); } - log.log(LogLevel.DEBUG, () -> + log.log(Level.FINE, () -> String.format("Metric retrieval for %d nodes took %d milliseconds", hosts.size(), System.currentTimeMillis() - startTime) ); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java index 4fe2ec129d3..6366576e163 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/model/LbServicesProducer.java @@ -9,9 +9,7 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import java.util.Collections; import java.util.Comparator; @@ -34,17 +32,14 @@ public class LbServicesProducer implements LbServicesConfig.Producer { private final Map<TenantName, Set<ApplicationInfo>> models; private final Zone zone; - private final BooleanFlag nginxUpstreamProxyProtocol; public LbServicesProducer(Map<TenantName, Set<ApplicationInfo>> models, Zone zone, FlagSource flagSource) { this.models = models; this.zone = zone; - this.nginxUpstreamProxyProtocol = Flags.NGINX_UPSTREAM_PROXY_PROTOCOL.bindTo(flagSource); } @Override public void getConfig(LbServicesConfig.Builder builder) { - builder.nginxUpstreamProxyProtocol(nginxUpstreamProxyProtocol.value()); models.keySet().stream() .sorted() .forEach(tenant -> { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index f62d065960d..0b83e927c39 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -16,7 +16,7 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ServerCache; @@ -89,7 +89,7 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { Version wantedNodeVespaVersion, Optional<AllocatedHosts> ignored, // Ignored since we have this in the app package for activated models Instant now) { - log.log(LogLevel.DEBUG, String.format("Loading model version %s for session %s application %s", + log.log(Level.FINE, String.format("Loading model version %s for session %s application %s", modelFactory.version(), appGeneration, applicationId)); ModelContext.Properties modelContextProperties = createModelContextProperties(applicationId); Provisioned provisioned = new Provisioned(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 5cac74907d4..455731d9cb6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -16,7 +16,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.TransientException; import com.yahoo.config.provision.Zone; import com.yahoo.lang.SettableOptional; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.UnknownVespaVersionException; import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; @@ -78,7 +78,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { ApplicationPackage applicationPackage, SettableOptional<AllocatedHosts> allocatedHosts, Instant now) { - log.log(LogLevel.DEBUG, "Will build models for " + applicationId); + log.log(Level.FINE, "Will build models for " + applicationId); Set<Version> versions = modelFactoryRegistry.allVersions(); // If the application specifies a major, skip models on a newer major @@ -118,19 +118,19 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { } catch (RuntimeException e) { if (shouldSkipCreatingMajorVersionOnError(majorVersions, majorVersion)) { - log.log(LogLevel.INFO, applicationId + ": Skipping major version " + majorVersion, e); + log.log(Level.INFO, applicationId + ": Skipping major version " + majorVersion, e); } else { if (e instanceof NullPointerException || e instanceof NoSuchElementException | e instanceof UncheckedTimeoutException) { - log.log(LogLevel.WARNING, "Unexpected error when building model ", e); + log.log(Level.WARNING, "Unexpected error when building model ", e); throw new InternalServerException(applicationId + ": Error loading model", e); } else { - log.log(LogLevel.WARNING, "Input error when building model ", e); + log.log(Level.WARNING, "Input error when building model ", e); throw new IllegalArgumentException(applicationId + ": Error loading model", e); } } } } - log.log(LogLevel.DEBUG, "Done building models for " + applicationId + ". Built models for versions " + + log.log(Level.FINE, "Done building models for " + applicationId + ". Built models for versions " + allApplicationModels.stream() .map(result -> result.getModel().version()) .map(Version::toFullString) @@ -197,9 +197,9 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { // allow failure to create old config models if there is a validation override that allow skipping old // config models (which is always true for manually deployed zones) if (allApplicationVersions.size() > 0 && allApplicationVersions.get(0).getModel().skipOldConfigModels(now)) - log.log(LogLevel.INFO, applicationId + ": Skipping old version (due to validation override)"); + log.log(Level.INFO, applicationId + ": Skipping old version (due to validation override)"); else { - log.log(LogLevel.ERROR, applicationId + ": Failed to build version " + version); + log.log(Level.SEVERE, applicationId + ": Failed to build version " + version); throw e; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index 82648d40d60..b8e7d68ea5e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -19,7 +19,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.component.Version; import com.yahoo.config.provision.DockerImage; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.host.HostValidator; @@ -90,7 +90,7 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P Optional<AllocatedHosts> allocatedHosts, Instant now) { Version modelVersion = modelFactory.version(); - log.log(LogLevel.DEBUG, "Building model " + modelVersion + " for " + applicationId); + log.log(Level.FINE, "Building model " + modelVersion + " for " + applicationId); FileDistributionProvider fileDistributionProvider = fileDistributionFactory.createProvider(context.getServerDBSessionDir()); // Use empty on non-hosted systems, use already allocated hosts if available, create connection to a host provisioner otherwise @@ -110,12 +110,12 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P modelVersion, wantedNodeVespaVersion); - log.log(LogLevel.DEBUG, "Create and validate model " + modelVersion + " for " + applicationId); + log.log(Level.FINE, "Create and validate model " + modelVersion + " for " + applicationId); ValidationParameters validationParameters = new ValidationParameters(params.ignoreValidationErrors() ? IgnoreValidationErrors.TRUE : IgnoreValidationErrors.FALSE); ModelCreateResult result = modelFactory.createAndValidateModel(modelContext, validationParameters); validateModelHosts(context.getHostValidator(), applicationId, result.getModel()); - log.log(LogLevel.DEBUG, "Done building model " + modelVersion + " for " + applicationId); + log.log(Level.FINE, "Done building model " + modelVersion + " for " + applicationId); return new PreparedModelsBuilder.PreparedModelResult(modelVersion, result.getModel(), fileDistributionProvider, result.getConfigChangeActions()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java index 249c8639ea9..8b9580c9688 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/monitoring/Metrics.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.docproc.jdisc.metric.NullMetric; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.statistics.Statistics; import com.yahoo.statistics.Counter; @@ -57,7 +57,7 @@ public class Metrics extends AbstractComponent implements MetricUpdaterFactory, procTimeCounter = createCounter("procTime", statistics); if (createZkMetricUpdater) { - log.log(LogLevel.DEBUG, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds"); + log.log(Level.FINE, "Metric update interval is " + healthMonitorConfig.snapshot_interval() + " seconds"); long intervalMs = (long) (healthMonitorConfig.snapshot_interval() * 1000); executorService = Optional.of(new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("configserver-metrics"))); executorService.get().scheduleAtFixedRate(this, 20000, intervalMs, TimeUnit.MILLISECONDS); @@ -134,9 +134,9 @@ public class Metrics extends AbstractComponent implements MetricUpdaterFactory, @Override public void run() { for (MetricUpdater metricUpdater : metricUpdaters.values()) { - log.log(LogLevel.DEBUG, "Running metric updater for static values for " + metricUpdater.getDimensions()); + log.log(Level.FINE, "Running metric updater for static values for " + metricUpdater.getDimensions()); for (Map.Entry<String, Number> fixedMetric : metricUpdater.getStaticMetrics().entrySet()) { - log.log(LogLevel.DEBUG, "Setting " + fixedMetric.getKey()); + log.log(Level.FINE, "Setting " + fixedMetric.getKey()); metric.set(fixedMetric.getKey(), fixedMetric.getValue(), metricUpdater.getMetricContext()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java index 32a641a3bc7..eb14947f73f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java @@ -5,7 +5,7 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.config.provision.Provisioner; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.Optional; import java.util.logging.Logger; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java index f58c6cfdee5..0d8b964a6d3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/DelayedConfigResponses.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.jrt.Target; import com.yahoo.jrt.TargetWatcher; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; import com.yahoo.vespa.config.server.GetConfigContext; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; @@ -88,8 +88,8 @@ public class DelayedConfigResponses { removeFromQueue(); removeWatcher(); rpcServer.addToRequestQueue(request, true, null); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, logPre()+"DelayedConfigResponse. putting on queue: " + request.getShortDescription()); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, logPre()+"DelayedConfigResponse. putting on queue: " + request.getShortDescription()); } } @@ -169,15 +169,15 @@ public class DelayedConfigResponses { */ final void delayResponse(JRTServerConfigRequest request, GetConfigContext context) { if (request.isDelayedResponse()) { - log.log(LogLevel.DEBUG, context.logPre()+"Request already delayed"); + log.log(Level.FINE, context.logPre()+"Request already delayed"); } else { createQueueIfNotExists(context); BlockingQueue<DelayedConfigResponse> delayedResponsesQueue = delayedResponses.get(context.applicationId()); DelayedConfigResponse response = new DelayedConfigResponse(request, delayedResponsesQueue, context.applicationId()); request.setDelayedResponse(true); try { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, context.logPre()+"Putting on delayedRequests queue (" + delayedResponsesQueue.size() + " elements): " + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, context.logPre()+"Putting on delayedRequests queue (" + delayedResponsesQueue.size() + " elements): " + response.getRequest().getShortDescription()); } // Config will be resolved in the run() method of DelayedConfigResponse, @@ -185,7 +185,7 @@ public class DelayedConfigResponses { response.schedule(Math.max(0, request.getTimeout())); metricDelayedResponses(context.applicationId(), delayedResponsesQueue.size()); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, context.logPre()+"Interrupted when putting on delayed requests queue."); + log.log(Level.WARNING, context.logPre()+"Interrupted when putting on delayed requests queue."); } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 48580dbc6f4..7809000695a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -6,7 +6,7 @@ import com.yahoo.collections.Pair; import com.yahoo.config.provision.TenantName; import com.yahoo.component.Version; import com.yahoo.jrt.Request; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.net.HostName; import com.yahoo.vespa.config.ConfigPayload; import com.yahoo.vespa.config.ErrorCode; @@ -49,7 +49,7 @@ class GetConfigProcessor implements Runnable { private void respond(JRTServerConfigRequest request) { Request req = request.getRequest(); if (req.isError()) { - Level logLevel = (req.errorCode() == ErrorCode.APPLICATION_NOT_LOADED) ? LogLevel.DEBUG : LogLevel.INFO; + Level logLevel = (req.errorCode() == ErrorCode.APPLICATION_NOT_LOADED) ? Level.FINE : Level.INFO; log.log(logLevel, logPre + req.errorMessage()); } rpcServer.respond(request); @@ -71,7 +71,7 @@ class GetConfigProcessor implements Runnable { //Request has already been detached if ( ! request.validateParameters()) { // Error code is set in verifyParameters if parameters are not OK. - log.log(LogLevel.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); + log.log(Level.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); respond(request); return null; } @@ -164,7 +164,7 @@ class GetConfigProcessor implements Runnable { } private void returnEmpty(JRTServerConfigRequest request) { - log.log(LogLevel.DEBUG, () -> "Returning empty sentinel config for request from " + request.getClientHostName()); + log.log(Level.FINE, () -> "Returning empty sentinel config for request from " + request.getClientHostName()); ConfigPayload emptyPayload = ConfigPayload.empty(); String configMd5 = ConfigUtils.getMd5(emptyPayload); ConfigResponse config = SlimeConfigResponse.fromConfigPayload(emptyPayload, 0, false, configMd5); @@ -173,12 +173,12 @@ class GetConfigProcessor implements Runnable { } static boolean logDebug(Trace trace) { - return trace.shouldTrace(RpcServer.TRACELEVEL_DEBUG) || log.isLoggable(LogLevel.DEBUG); + return trace.shouldTrace(RpcServer.TRACELEVEL_DEBUG) || log.isLoggable(Level.FINE); } private void debugLog(Trace trace, String message) { if (logDebug(trace)) { - log.log(LogLevel.DEBUG, logPre + message); + log.log(Level.FINE, logPre + message); trace.trace(RpcServer.TRACELEVEL_DEBUG, logPre + message); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 851ad166c2d..3cad8def541 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -21,7 +21,7 @@ import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ErrorCode; import com.yahoo.vespa.config.JRTMethods; import com.yahoo.vespa.config.protocol.ConfigResponse; @@ -156,8 +156,8 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { * Uses the template pattern to call methods in classes that extend RpcServer. */ private void getConfigV3(Request req) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, getConfigMethodName); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, getConfigMethodName); } req.detach(); rpcAuthorizer.authorizeConfigRequest(req) @@ -182,7 +182,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { @Override public void run() { - log.log(LogLevel.INFO, "Rpc server will listen on port " + spec.port()); + log.log(Level.INFO, "Rpc server will listen on port " + spec.port()); try { Acceptor acceptor = supervisor.listen(spec); isRunning = true; @@ -262,8 +262,8 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { void configReloaded(ApplicationId applicationId) { List<DelayedConfigResponses.DelayedConfigResponse> responses = delayedConfigResponses.drainQueue(applicationId); String logPre = TenantRepository.logPre(applicationId); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, logPre + "Start of configReload: " + responses.size() + " requests on delayed requests queue"); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, logPre + "Start of configReload: " + responses.size() + " requests on delayed requests queue"); } int responsesSent = 0; CompletionService<Boolean> completionService = new ExecutorCompletionService<>(executorService); @@ -273,15 +273,15 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { // Doing cancel here deals with the case where the timer is already running or has not run, so // there is no need for any extra check. if (delayedConfigResponse.cancel()) { - if (log.isLoggable(LogLevel.DEBUG)) { - logRequestDebug(LogLevel.DEBUG, logPre + "Timer cancelled for ", delayedConfigResponse.request); + if (log.isLoggable(Level.FINE)) { + logRequestDebug(Level.FINE, logPre + "Timer cancelled for ", delayedConfigResponse.request); } // Do not wait for this request if we were unable to execute if (addToRequestQueue(delayedConfigResponse.request, false, completionService)) { responsesSent++; } } else { - log.log(LogLevel.DEBUG, logPre + "Timer already cancelled or finished or never scheduled"); + log.log(Level.FINE, logPre + "Timer already cancelled or finished or never scheduled"); } } @@ -293,10 +293,10 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } } - log.log(LogLevel.DEBUG, logPre + "Finished reloading " + responsesSent + " requests"); + log.log(Level.FINE, logPre + "Finished reloading " + responsesSent + " requests"); } - private void logRequestDebug(LogLevel level, String message, JRTServerConfigRequest request) { + private void logRequestDebug(Level level, String message, JRTServerConfigRequest request) { if (log.isLoggable(level)) { log.log(level, message + request.getShortDescription()); } @@ -304,7 +304,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { @Override public void hostsUpdated(TenantName tenant, Collection<String> newHosts) { - log.log(LogLevel.DEBUG, "Updating hosts in tenant host registry '" + hostRegistry + "' with " + newHosts); + log.log(Level.FINE, "Updating hosts in tenant host registry '" + hostRegistry + "' with " + newHosts); hostRegistry.update(tenant, newHosts); } @@ -321,8 +321,8 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { } public void respond(JRTServerConfigRequest request) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Trace at request return:\n" + request.getRequestTrace().toString()); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Trace at request return:\n" + request.getRequestTrace().toString()); } request.getRequest().returnRequest(); } @@ -338,8 +338,8 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { if (tenant == null) { if (GetConfigProcessor.logDebug(trace)) { String message = "Did not find tenant for host '" + hostname + "', using " + TenantName.defaultName(); - log.log(LogLevel.DEBUG, message); - log.log(LogLevel.DEBUG, "hosts in host registry: " + hostRegistry.getAllHosts()); + log.log(Level.FINE, message); + log.log(Level.FINE, "hosts in host registry: " + hostRegistry.getAllHosts()); trace.trace(6, message); } return Optional.empty(); @@ -398,7 +398,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { String msg = TenantRepository.logPre(tenant) + "Unable to find request handler for tenant. Requested from host '" + request.getClientHostName() + "'"; metrics.incUnknownHostRequests(); trace.trace(TRACELEVEL, msg); - log.log(LogLevel.WARNING, msg); + log.log(Level.WARNING, msg); return null; } RequestHandler handler = requestHandler.get(); @@ -420,7 +420,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { @Override public void onTenantDelete(TenantName tenant) { - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant)+"Tenant deleted, removing request handler and cleaning host registry"); + log.log(Level.FINE, TenantRepository.logPre(tenant)+"Tenant deleted, removing request handler and cleaning host registry"); tenantProviders.remove(tenant); hostRegistry.removeHostsForKey(tenant); } @@ -433,7 +433,7 @@ public class RpcServer implements Runnable, ReloadListener, TenantListener { @Override public void onTenantCreate(TenantName tenant, TenantHandlerProvider tenantHandlerProvider) { - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant)+"Tenant created, adding request handler"); + log.log(Level.FINE, TenantRepository.logPre(tenant)+"Tenant created, adding request handler"); tenantProviders.put(tenant, tenantHandlerProvider); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java index 8d17a32d102..3dbb3fd11e3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.security.NodeIdentifierException; import com.yahoo.config.provision.security.NodeIdentity; import com.yahoo.jrt.Request; import com.yahoo.jrt.SecurityContext; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.tls.MixedMode; import com.yahoo.security.tls.TransportSecurityUtils; import com.yahoo.vespa.config.ConfigKey; @@ -84,7 +84,7 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { try { getPeerIdentity(request) .ifPresent(peerIdentity -> authorizer.accept(request, peerIdentity)); - log.log(LogLevel.DEBUG, () -> String.format("Authorization succeeded for request '%s' from '%s'", + log.log(Level.FINE, () -> String.format("Authorization succeeded for request '%s' from '%s'", request.methodName(), request.target().toString())); } catch (Throwable t) { handleAuthorizationFailure(request, t); @@ -156,9 +156,9 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { boolean isAuthorizationException = throwable instanceof AuthorizationException; String errorMessage = String.format("For request '%s' from '%s': %s", request.methodName(), request.target().toString(), throwable.getMessage()); if (!isAuthorizationException || ((AuthorizationException) throwable).type() != Type.SILENT) { - log.log(LogLevel.INFO, errorMessage); + log.log(Level.INFO, errorMessage); } - log.log(LogLevel.DEBUG, throwable, throwable::getMessage); + log.log(Level.FINE, throwable, throwable::getMessage); JrtErrorCode error = isAuthorizationException ? JrtErrorCode.UNAUTHORIZED : JrtErrorCode.AUTHORIZATION_FAILED; request.setError(error.code, errorMessage); request.returnRequest(); @@ -180,7 +180,7 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { } try { NodeIdentity identity = nodeIdentifier.identifyNode(certChain); - log.log(LogLevel.DEBUG, () -> String.format("Client '%s' identified as %s", request.target().toString(), identity.toString())); + log.log(Level.FINE, () -> String.format("Client '%s' identified as %s", request.target().toString(), identity.toString())); return Optional.of(identity); } catch (NodeIdentifierException e) { throw new AuthorizationException("Failed to identity peer: " + e.getMessage(), e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java index ad7a5116ac4..97c4530e447 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionRepo.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.config.server.session; import com.yahoo.config.provision.TenantName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.config.server.GlobalComponentRegistry; @@ -71,14 +71,14 @@ public class LocalSessionRepo extends SessionRepo<LocalSession> { try { addSession(loader.loadSession(Long.parseLong(session.getName()))); } catch (IllegalArgumentException e) { - log.log(LogLevel.WARNING, "Could not load session '" + + log.log(Level.WARNING, "Could not load session '" + session.getAbsolutePath() + "':" + e.getMessage() + ", skipping it."); } } } public void purgeOldSessions() { - log.log(LogLevel.DEBUG, "Purging old sessions"); + log.log(Level.FINE, "Purging old sessions"); try { List<LocalSession> sessions = new ArrayList<>(listSessions()); for (LocalSession candidate : sessions) { @@ -88,9 +88,9 @@ public class LocalSessionRepo extends SessionRepo<LocalSession> { } // Make sure to catch here, to avoid executor just dying in case of issues ... } catch (Throwable e) { - log.log(LogLevel.WARNING, "Error when purging old sessions ", e); + log.log(Level.WARNING, "Error when purging old sessions ", e); } - log.log(LogLevel.DEBUG, "Done purging old sessions"); + log.log(Level.FINE, "Done purging old sessions"); } private boolean hasExpired(LocalSession candidate) { @@ -103,7 +103,7 @@ public class LocalSessionRepo extends SessionRepo<LocalSession> { void deleteSession(LocalSession session) { long sessionId = session.getSessionId(); - log.log(LogLevel.DEBUG, "Deleting local session " + sessionId); + log.log(Level.FINE, "Deleting local session " + sessionId); LocalSessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); if (watcher != null) watcher.close(); removeSession(sessionId); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java index 3ac98f9ac4c..662094fc0ca 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSessionStateWatcher.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.recipes.cache.ChildData; @@ -37,11 +37,11 @@ public class LocalSessionStateWatcher { // Will delete session if it exists in local session repo private void sessionChanged(Session.Status status) { long sessionId = session.getSessionId(); - log.log(status == Session.Status.DELETE ? LogLevel.INFO : LogLevel.DEBUG, + log.log(status == Session.Status.DELETE ? Level.INFO : Level.FINE, session.logPre() + "Session change: Local session " + sessionId + " changed status to " + status); if (status.equals(Session.Status.DELETE) && localSessionRepo.getSession(sessionId) != null) { - log.log(LogLevel.DEBUG, session.logPre() + "Deleting session " + sessionId); + log.log(Level.FINE, session.logPre() + "Deleting session " + sessionId); localSessionRepo.deleteSession(session); } } @@ -54,7 +54,7 @@ public class LocalSessionStateWatcher { try { fileCache.close(); } catch (Exception e) { - log.log(LogLevel.WARNING, "Exception when closing watcher", e); + log.log(Level.WARNING, "Exception when closing watcher", e); } } @@ -66,7 +66,7 @@ public class LocalSessionStateWatcher { sessionChanged(Session.Status.parse(Utf8.toString(node.getData()))); } } catch (Exception e) { - log.log(LogLevel.WARNING, session.logPre() + "Error handling session changed for session " + getSessionId(), e); + log.log(Level.WARNING, session.logPre() + "Error handling session changed for session " + getSessionId(), e); } }); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 39c15474f0e..9da3ad2fd7a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -5,7 +5,7 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.lang.SettableOptional; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ReloadHandler; @@ -88,13 +88,13 @@ public class RemoteSession extends Session { void makeActive(ReloadHandler reloadHandler) { Curator.CompletionWaiter waiter = zooKeeperClient.getActiveWaiter(); - log.log(LogLevel.DEBUG, () -> logPre() + "Getting session from repo: " + getSessionId()); + log.log(Level.FINE, () -> logPre() + "Getting session from repo: " + getSessionId()); ApplicationSet app = ensureApplicationLoaded(); - log.log(LogLevel.DEBUG, () -> logPre() + "Reloading config for " + getSessionId()); + log.log(Level.FINE, () -> logPre() + "Reloading config for " + getSessionId()); reloadHandler.reloadConfig(app); - log.log(LogLevel.DEBUG, () -> logPre() + "Notifying " + waiter); + log.log(Level.FINE, () -> logPre() + "Notifying " + waiter); notifyCompletion(waiter); - log.log(LogLevel.INFO, logPre() + "Session activated: " + getSessionId()); + log.log(Level.INFO, logPre() + "Session activated: " + getSessionId()); } @Override @@ -108,9 +108,9 @@ public class RemoteSession extends Session { void confirmUpload() { Curator.CompletionWaiter waiter = zooKeeperClient.getUploadWaiter(); - log.log(LogLevel.DEBUG, "Notifying upload waiter for session " + getSessionId()); + log.log(Level.FINE, "Notifying upload waiter for session " + getSessionId()); notifyCompletion(waiter); - log.log(LogLevel.DEBUG, "Done notifying upload for session " + getSessionId()); + log.log(Level.FINE, "Done notifying upload for session " + getSessionId()); } private void notifyCompletion(Curator.CompletionWaiter completionWaiter) { @@ -124,7 +124,7 @@ public class RemoteSession extends Session { if (e.getCause().getClass() != KeeperException.NoNodeException.class) { throw e; } else { - log.log(LogLevel.INFO, "Not able to notify completion for session: " + getSessionId() + ", node has been deleted"); + log.log(Level.INFO, "Not able to notify completion for session: " + getSessionId() + ", node has been deleted"); } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index 936b9bdefda..6dade946e5d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -6,7 +6,7 @@ import com.google.common.collect.Multiset; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ReloadHandler; @@ -88,7 +88,7 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { if (session == null) continue; // Internal sessions not in synch with zk, continue Instant created = Instant.ofEpochSecond(session.getCreateTime()); if (sessionHasExpired(created, expiryTime)) { - log.log(LogLevel.INFO, "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); + log.log(Level.INFO, "Remote session " + sessionId + " for " + tenantName + " has expired, deleting it"); session.delete(); deleted++; } @@ -138,7 +138,7 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { * @param sessionId session id for the new session */ private void sessionAdded(long sessionId) { - log.log(LogLevel.DEBUG, () -> "Adding session to RemoteSessionRepo: " + sessionId); + log.log(Level.FINE, () -> "Adding session to RemoteSessionRepo: " + sessionId); try { RemoteSession session = remoteSessionFactory.createSession(sessionId); Path sessionPath = sessionsPath.append(String.valueOf(sessionId)); @@ -164,9 +164,9 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { private void loadSessionIfActive(RemoteSession session) { for (ApplicationId applicationId : applicationRepo.activeApplications()) { if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { - log.log(LogLevel.DEBUG, () -> "Found active application for session " + session.getSessionId() + " , loading it"); + log.log(Level.FINE, () -> "Found active application for session " + session.getSessionId() + " , loading it"); reloadHandler.reloadConfig(session.ensureApplicationLoaded()); - log.log(LogLevel.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); + log.log(Level.INFO, session.logPre() + "Application activated successfully: " + applicationId + " (generation " + session.getSessionId() + ")"); return; } } @@ -178,7 +178,7 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { directoryCache.close(); } } catch (Exception e) { - log.log(LogLevel.WARNING, "Exception when closing path cache", e); + log.log(Level.WARNING, "Exception when closing path cache", e); } finally { checkForRemovedSessions(new ArrayList<>()); } @@ -200,7 +200,7 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { @SuppressWarnings("unused") private void childEvent(CuratorFramework ignored, PathChildrenCacheEvent event) { zkWatcherExecutor.execute(() -> { - log.log(LogLevel.DEBUG, () -> "Got child event: " + event); + log.log(Level.FINE, () -> "Got child event: " + event); switch (event.getType()) { case CHILD_ADDED: sessionsChanged(); @@ -220,7 +220,7 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { for (long sessionId : sessionList) { RemoteSession session = getSession(sessionId); if (session == null) continue; // session might have been deleted after getting session list - log.log(LogLevel.DEBUG, () -> session.logPre() + "Confirming upload for session " + sessionId); + log.log(Level.FINE, () -> session.logPre() + "Confirming upload for session " + sessionId); session.confirmUpload(); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java index a3fad3d7322..eb4047e5fea 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionStateWatcher.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; @@ -43,11 +43,11 @@ public class RemoteSessionStateWatcher { } private void sessionChanged(Session.Status status) { - log.log(LogLevel.DEBUG, session.logPre() + "Session change: Remote session " + session.getSessionId() + " changed status to " + status); + log.log(Level.FINE, session.logPre() + "Session change: Remote session " + session.getSessionId() + " changed status to " + status); // valid for NEW -> PREPARE transitions, not ACTIVATE -> PREPARE. if (status.equals(Session.Status.PREPARE)) { - log.log(LogLevel.DEBUG, session.logPre() + "Loading prepared session: " + session.getSessionId()); + log.log(Level.FINE, session.logPre() + "Loading prepared session: " + session.getSessionId()); session.loadPrepared(); } else if (status.equals(Session.Status.ACTIVATE)) { session.makeActive(reloadHandler); @@ -66,7 +66,7 @@ public class RemoteSessionStateWatcher { try { fileCache.close(); } catch (Exception e) { - log.log(LogLevel.WARNING, "Exception when closing watcher", e); + log.log(Level.WARNING, "Exception when closing watcher", e); } } @@ -78,7 +78,7 @@ public class RemoteSessionStateWatcher { sessionChanged(Session.Status.parse(Utf8.toString(node.getData()))); } } catch (Exception e) { - log.log(LogLevel.WARNING, session.logPre() + "Error handling session changed for session " + getSessionId(), e); + log.log(Level.WARNING, session.logPre() + "Error handling session changed for session " + getSessionId(), e); metrics.incSessionChangeErrors(); } }); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index 5ae1289033d..58b643ec82c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -6,7 +6,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.application.provider.*; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.*; import com.yahoo.config.provision.ApplicationId; @@ -105,16 +105,16 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { SessionZooKeeperClient sessionZKClient, TimeoutBudget timeoutBudget, Clock clock) { - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Creating session " + sessionId + " in ZooKeeper"); + log.log(Level.FINE, TenantRepository.logPre(tenant) + "Creating session " + sessionId + " in ZooKeeper"); sessionZKClient.createNewSession(clock.instant().toEpochMilli(), TimeUnit.MILLISECONDS); - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Creating upload waiter for session " + sessionId); + log.log(Level.FINE, TenantRepository.logPre(tenant) + "Creating upload waiter for session " + sessionId); Curator.CompletionWaiter waiter = sessionZKClient.getUploadWaiter(); - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Done creating upload waiter for session " + sessionId); + log.log(Level.FINE, TenantRepository.logPre(tenant) + "Done creating upload waiter for session " + sessionId); SessionContext context = new SessionContext(applicationPackage, sessionZKClient, getSessionAppDir(sessionId), applicationRepo, hostRegistry, flagSource); LocalSession session = new LocalSession(tenant, sessionId, sessionPreparer, context); - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Waiting on upload waiter for session " + sessionId); + log.log(Level.FINE, TenantRepository.logPre(tenant) + "Waiting on upload waiter for session " + sessionId); waiter.awaitCompletion(timeoutBudget.timeLeft()); - log.log(LogLevel.DEBUG, TenantRepository.logPre(tenant) + "Done waiting on upload waiter for session " + sessionId); + log.log(Level.FINE, TenantRepository.logPre(tenant) + "Done waiting on upload waiter for session " + sessionId); return session; } @@ -127,7 +127,7 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { ApplicationId existingApplicationId = existingSession.getApplicationId(); long activeSessionId = getActiveSessionId(existingApplicationId); - logger.log(LogLevel.DEBUG, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId); + logger.log(Level.FINE, "Create new session for application id '" + existingApplicationId + "' from existing active session " + activeSessionId); LocalSession session = create(existingApp, existingApplicationId, activeSessionId, internalRedeploy, timeoutBudget); // Note: Needs to be kept in sync with calls in SessionPreparer.writeStateToZooKeeper() session.setApplicationId(existingApplicationId); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 535ab8eaf86..00cafb77f6d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -23,7 +23,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.lang.SettableOptional; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.ConfigServerSpec; import com.yahoo.vespa.config.server.application.ApplicationSet; @@ -122,7 +122,7 @@ public class SessionPreparer { preparation.writeContainerEndpointsZK(); preparation.distribute(); } - log.log(LogLevel.DEBUG, () -> "time used " + params.getTimeoutBudget().timesUsed() + + log.log(Level.FINE, () -> "time used " + params.getTimeoutBudget().timesUsed() + " : " + params.getApplicationId()); return preparation.result(); } @@ -244,7 +244,7 @@ public class SessionPreparer { } void writeStateZK() { - log.log(LogLevel.DEBUG, "Writing application package state to zookeeper"); + log.log(Level.FINE, "Writing application package state to zookeeper"); writeStateToZooKeeper(context.getSessionZooKeeperClient(), applicationPackage, applicationId, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 64d4be680b8..3a248a5c3b9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeFlavors; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.transaction.NestedTransaction; @@ -83,7 +83,7 @@ public class SessionZooKeeperClient { String data = configCurator.getData(sessionStatusPath.getAbsolute()); return Session.Status.parse(data); } catch (Exception e) { - log.log(LogLevel.INFO, "Unable to read session status, assuming it was deleted"); + log.log(Level.INFO, "Unable to read session status, assuming it was deleted"); return Session.Status.NONE; } } @@ -129,13 +129,13 @@ public class SessionZooKeeperClient { public void delete(NestedTransaction transaction ) { try { - log.log(LogLevel.DEBUG, "Deleting " + sessionPath.getAbsolute()); + log.log(Level.FINE, "Deleting " + sessionPath.getAbsolute()); CuratorTransaction curatorTransaction = new CuratorTransaction(curator); CuratorOperations.deleteAll(sessionPath.getAbsolute(), curator).forEach(curatorTransaction::add); transaction.add(curatorTransaction); transaction.commit(); } catch (RuntimeException e) { - log.log(LogLevel.INFO, "Error deleting session (" + sessionPath.getAbsolute() + ") from zookeeper", e); + log.log(Level.INFO, "Error deleting session (" + sessionPath.getAbsolute() + ") from zookeeper", e); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java index 9842c90fff1..b249211081f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantDebugger.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.curator.Curator; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.TreeCache; @@ -32,7 +32,7 @@ public class TenantDebugger implements TreeCacheListener { case NODE_ADDED: case NODE_REMOVED: case NODE_UPDATED: - log.log(LogLevel.INFO, event.toString()); + log.log(Level.INFO, event.toString()); break; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 2e09d830783..04deed40c54 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -7,7 +7,7 @@ import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; @@ -121,10 +121,10 @@ public class TenantRepository { } else { this.directoryCache = Optional.empty(); } - log.log(LogLevel.DEBUG, "Creating all tenants"); + log.log(Level.FINE, "Creating all tenants"); bootstrapTenants(); notifyTenantsLoaded(); - log.log(LogLevel.DEBUG, "All tenants created"); + log.log(Level.FINE, "All tenants created"); checkForRemovedApplicationsService.scheduleWithFixedDelay(this::removeUnusedApplications, checkForRemovedApplicationsInterval.getSeconds(), checkForRemovedApplicationsInterval.getSeconds(), @@ -153,7 +153,7 @@ public class TenantRepository { /** Public for testing. */ public synchronized void updateTenants() { Set<TenantName> allTenants = readTenantsFromZooKeeper(curator); - log.log(LogLevel.DEBUG, "Create tenants, tenants found in zookeeper: " + allTenants); + log.log(Level.FINE, "Create tenants, tenants found in zookeeper: " + allTenants); for (TenantName tenantName : Set.copyOf(tenants.keySet())) if ( ! allTenants.contains(tenantName)) zkWatcherExecutor.execute(tenantName, () -> closeTenant(tenantName)); @@ -175,10 +175,10 @@ public class TenantRepository { try { f.getValue().get(); } catch (ExecutionException e) { - log.log(LogLevel.WARNING, "Failed to create tenant " + tenantName, e); + log.log(Level.WARNING, "Failed to create tenant " + tenantName, e); failed.add(tenantName); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, "Interrupted while creating tenant '" + tenantName + "'", e); + log.log(Level.WARNING, "Interrupted while creating tenant '" + tenantName + "'", e); } } @@ -203,7 +203,7 @@ public class TenantRepository { TenantName tenantName = builder.getTenantName(); if (tenants.containsKey(tenantName)) return; - log.log(LogLevel.INFO, "Creating tenant '" + tenantName + "'"); + log.log(Level.INFO, "Creating tenant '" + tenantName + "'"); Tenant tenant = builder.build(); notifyNewTenant(tenant); tenants.putIfAbsent(tenantName, tenant); @@ -280,7 +280,7 @@ public class TenantRepository { if ( ! tenants.containsKey(name)) throw new IllegalArgumentException("Deleting '" + name + "' failed, tenant does not exist"); - log.log(LogLevel.INFO, "Deleting tenant '" + name + "'"); + log.log(Level.INFO, "Deleting tenant '" + name + "'"); tenants.get(name).delete(); } @@ -289,7 +289,7 @@ public class TenantRepository { if (tenant == null) throw new IllegalArgumentException("Closing '" + name + "' failed, tenant does not exist"); - log.log(LogLevel.INFO, "Closing tenant '" + name + "'"); + log.log(Level.INFO, "Closing tenant '" + name + "'"); notifyRemovedTenant(name); tenant.close(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java index 2f9a5eb9277..1fb09b993fb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java @@ -6,7 +6,7 @@ import com.yahoo.concurrent.StripedExecutor; import com.yahoo.config.FileReference; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.protocol.ConfigResponse; @@ -80,8 +80,8 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host @Override public ConfigResponse resolveConfig(ApplicationId appId, GetConfigRequest req, Optional<Version> vespaVersion) { Application application = getApplication(appId, vespaVersion); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, TenantRepository.logPre(appId) + "Resolving for tenant '" + tenant + "' with handler for application '" + application + "'"); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, TenantRepository.logPre(appId) + "Resolving for tenant '" + tenant + "' with handler for application '" + application + "'"); } return application.resolveConfig(req, responseFactory); } @@ -138,7 +138,7 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host public void removeApplicationsExcept(Set<ApplicationId> applications) { for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) { if ( ! applications.contains(activeApplication)) { - log.log(LogLevel.INFO, "Will remove deleted application " + activeApplication.toShortString()); + log.log(Level.INFO, "Will remove deleted application " + activeApplication.toShortString()); removeApplication(activeApplication); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java index f07327517f8..f3f9c914be8 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ConfigCurator.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.zookeeper; import com.google.inject.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.curator.Curator; @@ -70,7 +70,7 @@ public class ConfigCurator { private ConfigCurator(Curator curator, int maxNodeSize) { this.curator = curator; this.maxNodeSize = maxNodeSize; - log.log(LogLevel.CONFIG, "Using jute max buffer size " + this.maxNodeSize); + log.log(Level.CONFIG, "Using jute max buffer size " + this.maxNodeSize); testZkConnection(); } @@ -310,7 +310,7 @@ public class ConfigCurator { curator.framework().checkExists().forPath("/dummy"); } catch (Exception e) { - log.log(LogLevel.ERROR, "Unable to contact ZooKeeper on " + curator.connectionSpec() + + log.log(Level.SEVERE, "Unable to contact ZooKeeper on " + curator.connectionSpec() + ". Please verify for all configserver nodes that " + "VESPA_CONFIGSERVERS points to the correct configserver(s), " + "the same configserver(s) as in services.xml, and that they are started. " + diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java index 773888a8a03..389649f4680 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/InitializedCounter.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.zookeeper; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.curator.recipes.CuratorCounter; import java.util.ArrayList; @@ -27,7 +27,7 @@ public class InitializedCounter { } private void initializeCounterValue(Long latestSessionId) { - log.log(LogLevel.DEBUG, "path=" + sessionsDirPath + ", current=" + latestSessionId); + log.log(Level.FINE, "path=" + sessionsDirPath + ", current=" + latestSessionId); if (latestSessionId != null) { counter.initialize(latestSessionId); } else { @@ -45,7 +45,7 @@ public class InitializedCounter { try { return configCurator.exists(appsPath); } catch (Exception e) { - log.log(LogLevel.WARNING, e.getMessage()); + log.log(Level.WARNING, e.getMessage()); return false; } } @@ -64,7 +64,7 @@ public class InitializedCounter { newestGeneration = Collections.max(getDeployedApplicationGenerations(configCurator, appsPath)); } } catch (Exception e) { - log.log(LogLevel.WARNING, "Could not get newest application generation from Zookeeper"); + log.log(Level.WARNING, "Could not get newest application generation from Zookeeper"); } return newestGeneration; } @@ -79,7 +79,7 @@ public class InitializedCounter { } } } catch (RuntimeException e) { - log.log(LogLevel.WARNING, "Could not get application generations from Zookeeper"); + log.log(Level.WARNING, "Could not get application generations from Zookeeper"); } return generations; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java index ad0d4a2e540..34cf6e345a0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationFile.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.path.Path; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.util.ConfigUtils; import java.io.*; @@ -150,7 +150,7 @@ class ZKApplicationFile extends ApplicationFile { StringWriter writer = new StringWriter(); try { mapper.writeValue(writer, new MetaData(status, input == null ? "" : ConfigUtils.getMd5(input))); - log.log(LogLevel.DEBUG, "Writing meta file to " + metaPath); + log.log(Level.FINE, "Writing meta file to " + metaPath); zkApp.putData(metaPath, writer.toString()); } catch (IOException e) { throw new RuntimeException("Error writing meta file to " + metaPath, e); @@ -159,7 +159,7 @@ class ZKApplicationFile extends ApplicationFile { public MetaData getMetaData() { String metaPath = getZKPath(getMetaPath()); - log.log(LogLevel.DEBUG, "Getting metadata for " + metaPath); + log.log(Level.FINE, "Getting metadata for " + metaPath); if (!zkApp.exists(getZKPath(path))) { if (zkApp.exists(metaPath)) { return getMetaDataFromZk(metaPath); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java index d321edefe67..6aa72e3e672 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployHandlerLoggerTest.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.config.server.deploy; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; +import java.util.logging.Level; + import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; @@ -44,7 +46,7 @@ public class DeployHandlerLoggerTest { private void logMessages(DeployLogger logger) { logger.log(LogLevel.DEBUG, "foobar"); logger.log(LogLevel.SPAM, "foobar"); - logger.log(LogLevel.FINE, "baz"); - logger.log(LogLevel.WARNING, "baz"); + logger.log(Level.FINE, "baz"); + logger.log(Level.WARNING, "baz"); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index 84987bce32e..06db7018727 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -299,12 +299,6 @@ public class DeployTester { } @Override - @Deprecated // TODO: Remove after April 2020 - public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - return hostProvisioner.prepare(cluster, capacity.withGroups(groups), logger); - } - - @Override public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { return hostProvisioner.prepare(cluster, capacity, logger); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java index 89a901ad051..a1e252c8130 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/SessionHandlerTest.java @@ -18,7 +18,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -132,7 +132,7 @@ public class SessionHandlerTest { status = Session.Status.PREPARE; this.dockerImageRepository = params.dockerImageRepository(); if (doVerboseLogging) { - logger.log(LogLevel.DEBUG, "debuglog"); + logger.log(Level.FINE, "debuglog"); } return actions; } @@ -238,12 +238,6 @@ public class SessionHandlerTest { public Collection<HostSpec> lastHosts; @Override - @Deprecated // TODO: Remove after April 2020 - public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - throw new UnsupportedOperationException(); - } - - @Override public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { throw new UnsupportedOperationException(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index a3b0f3ec44a..cdf57b68814 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -20,7 +20,7 @@ import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.exception.LoadBalancerServiceException; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; @@ -176,7 +176,7 @@ public class SessionPreparerTest { final StringBuilder logged = new StringBuilder(); DeployLogger logger = (level, message) -> { System.out.println(level + ": "+message); - if (level.equals(LogLevel.WARNING) && message.contains("The host mytesthost is already in use")) logged.append("ok"); + if (level.equals(Level.WARNING) && message.contains("The host mytesthost is already in use")) logged.append("ok"); }; preparer.prepare(ctx, logger, new PrepareParams.Builder().build(), Optional.empty(), tenantPath, Instant.now()); assertEquals(logged.toString(), ""); @@ -342,12 +342,6 @@ public class SessionPreparerTest { private static class FailWithTransientExceptionProvisioner implements Provisioner { @Override - @Deprecated // TODO: Remove after April 2020 - public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception")); - } - - @Override public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { throw new LoadBalancerServiceException("Unable to create load balancer", new Exception("some internal exception")); } diff --git a/container-accesslogging/pom.xml b/container-accesslogging/pom.xml index 59dabba9efe..ba544e04fc8 100644 --- a/container-accesslogging/pom.xml +++ b/container-accesslogging/pom.xml @@ -51,6 +51,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespalog</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-all</artifactId> <scope>test</scope> diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 6d683c53984..82d34d88b99 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -171,6 +171,7 @@ "public void <init>()", "public void <init>(com.yahoo.container.handler.ThreadpoolConfig)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder maxthreads(int)", + "public com.yahoo.container.handler.ThreadpoolConfig$Builder queueSize(int)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder maxThreadExecutionTimeSeconds(int)", "public com.yahoo.container.handler.ThreadpoolConfig$Builder softStartSeconds(double)", "public final boolean dispatchGetConfig(com.yahoo.config.ConfigInstance$Producer)", @@ -210,6 +211,7 @@ "public static java.lang.String getDefVersion()", "public void <init>(com.yahoo.container.handler.ThreadpoolConfig$Builder)", "public int maxthreads()", + "public int queueSize()", "public int maxThreadExecutionTimeSeconds()", "public double softStartSeconds()" ], diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index d87b38e8b18..270decfeaeb 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -20,7 +20,7 @@ import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.jdisc.service.ServerProvider; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.osgi.OsgiImpl; import com.yahoo.osgi.OsgiWrapper; import com.yahoo.statistics.Statistics; @@ -57,7 +57,7 @@ public class HandlersConfigurerDi { ComponentRegistry<RequestHandler> requestHandlerRegistry, ComponentRegistry<ClientProvider> clientProviderRegistry, ComponentRegistry<ServerProvider> serverProviderRegistry) { - log.log(LogLevel.DEBUG, "RegistriesHack.init " + System.identityHashCode(this)); + log.log(Level.FINE, "RegistriesHack.init " + System.identityHashCode(this)); vespaContainer.setComponentRegistry(allComponents); vespaContainer.setRequestHandlerRegistry(requestHandlerRegistry); diff --git a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java index b427a58c9b7..0e786cfbc8f 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ThreadPoolProvider.java @@ -10,6 +10,7 @@ import com.yahoo.container.protect.ProcessTerminator; import com.yahoo.jdisc.Metric; import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; @@ -33,17 +34,30 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex private final ExecutorServiceWrapper threadpool; + private static BlockingQueue<Runnable> createQ(int queueSize, int maxThreads) { + return (queueSize == 0) + ? new SynchronousQueue<>(false) + : (queueSize < 0) + ? new ArrayBlockingQueue<>(maxThreads*4) + : new ArrayBlockingQueue<>(queueSize); + } + + private static int computeThreadPoolSize(int maxNumThreads) { + return (maxNumThreads <= 0) + ? Runtime.getRuntime().availableProcessors() * 4 + : maxNumThreads; + } @Inject public ThreadPoolProvider(ThreadpoolConfig threadpoolConfig, Metric metric) { this(threadpoolConfig, metric, new ProcessTerminator()); } public ThreadPoolProvider(ThreadpoolConfig threadpoolConfig, Metric metric, ProcessTerminator processTerminator) { + int maxNumThreads = computeThreadPoolSize(threadpoolConfig.maxthreads()); WorkerCompletionTimingThreadPoolExecutor executor = - new WorkerCompletionTimingThreadPoolExecutor(threadpoolConfig.maxthreads(), - threadpoolConfig.maxthreads(), + new WorkerCompletionTimingThreadPoolExecutor(maxNumThreads, maxNumThreads, 0L, TimeUnit.SECONDS, - new SynchronousQueue<>(false), + createQ(threadpoolConfig.queueSize(), maxNumThreads), ThreadFactoryFactory.getThreadFactory("threadpool"), metric); // Prestart needed, if not all threads will be created by the fist N tasks and hence they might also @@ -87,8 +101,9 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex /** * A service executor wrapper which emits metrics and * shuts down the vm when no workers are available for too long to avoid containers lingering in a blocked state. + * Package private for testing */ - private final static class ExecutorServiceWrapper extends ForwardingExecutorService { + final static class ExecutorServiceWrapper extends ForwardingExecutorService { private final WorkerCompletionTimingThreadPoolExecutor wrapped; private final Metric metric; @@ -160,8 +175,11 @@ public class ThreadPoolProvider extends AbstractComponent implements Provider<Ex } - /** A thread pool executor which maintains the last time a worker completed */ - private final static class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor { + /** + * A thread pool executor which maintains the last time a worker completed + * package private for testing + **/ + final static class WorkerCompletionTimingThreadPoolExecutor extends ThreadPoolExecutor { private static final String UNHANDLED_EXCEPTIONS_METRIC = "jdisc.thread_pool.unhandled_exceptions"; diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java index eceffb379aa..f2df5ddb7c5 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatusHandler.java @@ -18,7 +18,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8; import com.yahoo.vespa.defaults.Defaults; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java b/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java index dc3bd6fe8a8..9c48955bf4c 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ContentChannelOutputStream.java @@ -5,7 +5,7 @@ import com.yahoo.io.BufferChain; import com.yahoo.io.WritableByteTransmitter; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.yolean.Exceptions; import java.io.IOException; @@ -159,9 +159,9 @@ public class ContentChannelOutputStream extends OutputStream implements Writable Level logLevel; synchronized (failLock) { if (failed) { - logLevel = LogLevel.SPAM; + logLevel = Level.FINEST; } else { - logLevel = LogLevel.DEBUG; + logLevel = Level.FINE; } failed = true; } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java index 62dc81aa103..e0fc79b4be9 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/LoggingRequestHandler.java @@ -11,7 +11,7 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.http.server.jetty.AccessLoggingRequestHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.yolean.Exceptions; import java.io.IOException; @@ -166,7 +166,7 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { b.append("Timeout: ").append(timeoutInterval).append(" ms. "); b.append("Request string: ").append(req); - log.log(LogLevel.WARNING, "Slow execution. " + b); + log.log(Level.WARNING, "Slow execution. " + b); } private static class NullResponse extends ExtendedResponse { @@ -227,8 +227,8 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { public void failed(Throwable throwable) { long endTime = System.currentTimeMillis(); writeToLogs(endTime); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Got exception when writing to client: " + Exceptions.toMessageString(throwable)); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Got exception when writing to client: " + Exceptions.toMessageString(throwable)); } } @@ -308,7 +308,7 @@ public abstract class LoggingRequestHandler extends ThreadedHttpRequestHandler { logEntry.setHttpMethod(AccessLogUtil.getHttpMethod(httpRequest)); logEntry.setHttpVersion(AccessLogUtil.getHttpVersion(httpRequest)); } catch (Exception e) { - log.log(LogLevel.WARNING, "Could not populate the access log [" + fullRequest + "]", e); + log.log(Level.WARNING, "Could not populate the access log [" + fullRequest + "]", e); } } diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java index c58d49bf8c8..3a99ee7d0c6 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedHttpRequestHandler.java @@ -9,7 +9,7 @@ import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.UnsafeContentInputStream; import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.IOException; import java.nio.ByteBuffer; @@ -69,8 +69,8 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler @Override public final void handleRequest(Request request, BufferedContentChannel requestContent, ResponseHandler responseHandler) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "In " + this.getClass() + ".handleRequest()"); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "In " + this.getClass() + ".handleRequest()"); } com.yahoo.jdisc.http.HttpRequest jdiscRequest = asHttpRequest(request); HttpRequest httpRequest = new HttpRequest(jdiscRequest, new UnsafeContentInputStream(requestContent.toReadable())); @@ -83,7 +83,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler } catch (Exception e) { metric.add(UNHANDLED_EXCEPTIONS_METRIC, 1L, contextFor(request, Map.of("exception", e.getClass().getSimpleName()))); metric.add(RENDERING_ERRORS, 1, null); - log.log(LogLevel.ERROR, "Uncaught exception handling request", e); + log.log(Level.SEVERE, "Uncaught exception handling request", e); if (channel != null) { channel.setHttpResponse(null); channel.close(null); @@ -118,7 +118,7 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler catch (IOException e) { metric.add(RENDERING_ERRORS, 1, null); long time = System.currentTimeMillis() - startTime; - log.log(time < 900 ? LogLevel.INFO : LogLevel.WARNING, + log.log(time < 900 ? Level.INFO : Level.WARNING, "IO error while responding to " + " [" + request.getUri() + "] " + "(total time " + time + " ms) ", e); @@ -191,8 +191,8 @@ public abstract class ThreadedHttpRequestHandler extends ThreadedRequestHandler return responseHandler.handleResponse(httpResponse.getJdiscResponse()); } catch (Exception e) { metric.add(RENDERING_ERRORS, 1, null); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Error writing response to client - connection probably terminated " + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Error writing response to client - connection probably terminated " + "from client side.", e); } return new DevNullChannel(); // Ignore further operations on this diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java index 99732af9d31..34ba7676680 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/ThreadedRequestHandler.java @@ -15,7 +15,7 @@ import com.yahoo.jdisc.handler.OverloadException; import com.yahoo.jdisc.handler.ReadableContentChannel; import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.net.URI; import java.time.Duration; @@ -147,7 +147,7 @@ public abstract class ThreadedRequestHandler extends AbstractRequestHandler { currentFailureIntervalStartedMillis = 0L; numRejectedRequests = 0; } - log.log(LogLevel.WARNING, "Rejected " + numRejectedRequestsSnapshot + " requests on cause of no available worker threads."); + log.log(Level.WARNING, "Rejected " + numRejectedRequestsSnapshot + " requests on cause of no available worker threads."); } private void incrementRejectedRequests() { diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java index faa08402cdc..78b65622150 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/StateMonitor.java @@ -7,7 +7,7 @@ import com.yahoo.container.jdisc.config.HealthMonitorConfig; import com.yahoo.jdisc.Timer; import com.yahoo.jdisc.application.MetricConsumer; import com.yahoo.jdisc.core.SystemTimer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.Map; import java.util.TreeSet; @@ -78,7 +78,7 @@ public class StateMonitor extends AbstractComponent { public void status(Status status) { if (status != this.status) { - log.log(LogLevel.INFO, "Changing health status code from '" + this.status + "' to '" + status.name() + "'"); + log.log(Level.INFO, "Changing health status code from '" + this.status + "' to '" + status.name() + "'"); this.status = status; } } diff --git a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java index a18d1eee0fc..ba246f26639 100644 --- a/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java +++ b/container-core/src/main/java/com/yahoo/processing/rendering/AsynchronousSectionedRenderer.java @@ -7,7 +7,7 @@ import com.google.common.util.concurrent.SettableFuture; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.processing.Request; import com.yahoo.processing.Response; import com.yahoo.processing.execution.Execution; @@ -471,7 +471,7 @@ public abstract class AsynchronousSectionedRenderer<RESPONSE extends Response> e stream.close(); } catch (IOException e) { closeException = e; - logger.log(LogLevel.WARNING, "Exception caught while closing stream to client.", e); + logger.log(Level.WARNING, "Exception caught while closing stream to client.", e); } finally { if (failed != null) { success.setException(failed); @@ -527,9 +527,9 @@ public abstract class AsynchronousSectionedRenderer<RESPONSE extends Response> e try { render(); } catch (Exception e) { - Level level = LogLevel.WARNING; + Level level = Level.WARNING; if ((e instanceof IOException)) { - level = LogLevel.DEBUG; + level = Level.FINE; if ( ! clientClosed) { clientClosed = true; onClientClosed(); diff --git a/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java index f261c7611f1..94785819aa6 100644 --- a/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java +++ b/container-core/src/main/java/com/yahoo/restapi/JacksonJsonResponse.java @@ -5,7 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.IOException; import java.io.OutputStream; @@ -37,9 +37,9 @@ public class JacksonJsonResponse<T> extends HttpResponse { @Override public void render(OutputStream outputStream) throws IOException { - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { String json = jsonMapper.writeValueAsString(entity); - log.log(LogLevel.DEBUG, "Writing the following JSON to response output stream:\n" + json); + log.log(Level.FINE, "Writing the following JSON to response output stream:\n" + json); outputStream.write(json.getBytes()); } else { jsonMapper.writeValue(outputStream, entity); diff --git a/container-core/src/main/resources/configdefinitions/threadpool.def b/container-core/src/main/resources/configdefinitions/threadpool.def index 9bb9badd9b5..abc60f9f06d 100644 --- a/container-core/src/main/resources/configdefinitions/threadpool.def +++ b/container-core/src/main/resources/configdefinitions/threadpool.def @@ -2,8 +2,16 @@ namespace=container.handler +## Num ber of thread in the thread pool +## Setting it to 0 or negative number will cause it to be set to #cores * 4 maxthreads int default=500 +## max queue size +## There can be queueSize + maxthreads requests inflight concurrently +## The container will start replying 503 +## Negative value will cause it to set to maxthreads*4 +queueSize int default=0 + # The max time the container tolerates having no threads available before it shuts down to # get out of a bad state. This should be set a bit higher than the expected max execution # time of each request when in a state of overload, i.e about "worst case execution time*2" diff --git a/container-core/src/test/java/com/yahoo/container/handler/ThreadPoolProviderTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/ThreadPoolProviderTestCase.java index 918863f6dda..761ed40763c 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/ThreadPoolProviderTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/ThreadPoolProviderTestCase.java @@ -5,6 +5,7 @@ import static org.junit.Assert.fail; import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; import com.yahoo.container.protect.ProcessTerminator; import org.junit.Ignore; @@ -58,6 +59,39 @@ public class ThreadPoolProviderTestCase { fail("Pool did not reject tasks after shutdown."); } + private ThreadPoolExecutor createPool(int maxThreads, int queueSize) { + ThreadpoolConfig config = new ThreadpoolConfig(new ThreadpoolConfig.Builder().maxthreads(maxThreads).queueSize(queueSize)); + ThreadPoolProvider provider = new ThreadPoolProvider(config, Mockito.mock(Metric.class)); + ThreadPoolProvider.ExecutorServiceWrapper wrapper = (ThreadPoolProvider.ExecutorServiceWrapper) provider.get(); + ThreadPoolProvider.WorkerCompletionTimingThreadPoolExecutor executor = (ThreadPoolProvider.WorkerCompletionTimingThreadPoolExecutor)wrapper.delegate(); + return executor; + } + + @Test + public void testThatThreadPoolSizeFollowsConfig() { + ThreadPoolExecutor executor = createPool(3, 9); + assertEquals(3, executor.getMaximumPoolSize()); + assertEquals(9, executor.getQueue().remainingCapacity()); + } + @Test + public void testThatThreadPoolSizeAutoDetected() { + ThreadPoolExecutor executor = createPool(0, 0); + assertEquals(Runtime.getRuntime().availableProcessors()*4, executor.getMaximumPoolSize()); + assertEquals(0, executor.getQueue().remainingCapacity()); + } + @Test + public void testThatQueueSizeAutoDetected() { + ThreadPoolExecutor executor = createPool(3, -1); + assertEquals(3, executor.getMaximumPoolSize()); + assertEquals(executor.getMaximumPoolSize()*4, executor.getQueue().remainingCapacity()); + } + @Test + public void testThatThreadPoolSizeAndQueueSizeAutoDetected() { + ThreadPoolExecutor executor = createPool(0, -1); + assertEquals(Runtime.getRuntime().availableProcessors()*4, executor.getMaximumPoolSize()); + assertEquals(executor.getMaximumPoolSize()*4, executor.getQueue().remainingCapacity()); + } + private class FlipIt implements Runnable { public final Receiver<Boolean> didItRun = new Receiver<>(); diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java b/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java index 507ef6952b6..22e55eb5291 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/LoggingTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.container.jdisc; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -101,9 +101,9 @@ public class LoggingTestCase { stream.send(createData()); stream.send(createData()); stream.flush(); - assertNull(logChecker.errorCounter.get(LogLevel.INFO)); - assertEquals(1, logChecker.errorCounter.get(LogLevel.DEBUG).intValue()); - assertEquals(2, logChecker.errorCounter.get(LogLevel.SPAM).intValue()); + assertNull(logChecker.errorCounter.get(Level.INFO)); + assertEquals(1, logChecker.errorCounter.get(Level.FINE).intValue()); + assertEquals(2, logChecker.errorCounter.get(Level.FINEST).intValue()); } } diff --git a/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java b/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java index fde4ecfd33a..6ed9c2a2994 100644 --- a/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java +++ b/container-di/src/main/java/com/yahoo/container/di/ConfigRetriever.java @@ -16,7 +16,7 @@ import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; /** * @author Tony Vaagenes @@ -56,7 +56,7 @@ public final class ConfigRetriever { throw new IllegalArgumentException("Component config keys [" + componentConfigKeys + "] overlaps with bootstrap config keys [" + bootstrapKeys + "]"); - log.log(DEBUG, "getConfigs: " + componentConfigKeys); + log.log(FINE, "getConfigs: " + componentConfigKeys); Set<ConfigKey<? extends ConfigInstance>> allKeys = new HashSet<>(componentConfigKeys); allKeys.addAll(bootstrapKeys); setupComponentSubscriber(allKeys); @@ -84,7 +84,7 @@ public final class ConfigRetriever { throw new IllegalArgumentException( "Component config keys [" + componentConfigKeys + "] overlaps with bootstrap config keys [" + bootstrapKeys + "]"); } - log.log(DEBUG, "getConfigsOnce: " + componentConfigKeys); + log.log(FINE, "getConfigsOnce: " + componentConfigKeys); Set<ConfigKey<? extends ConfigInstance>> allKeys = new HashSet<>(componentConfigKeys); allKeys.addAll(bootstrapKeys); @@ -97,7 +97,7 @@ public final class ConfigRetriever { private Optional<ConfigSnapshot> getConfigsOptional(long leastGeneration, boolean restartOnRedeploy) { long newestComponentGeneration = componentSubscriber.waitNextGeneration(); - log.log(DEBUG, "getConfigsOptional: new component generation: " + newestComponentGeneration); + log.log(FINE, "getConfigsOptional: new component generation: " + newestComponentGeneration); // leastGeneration is only used to ensure newer generation when the previous generation was invalidated due to an exception if (newestComponentGeneration < leastGeneration) { @@ -106,13 +106,13 @@ public final class ConfigRetriever { return Optional.empty(); } else if (bootstrapSubscriber.generation() < newestComponentGeneration) { long newestBootstrapGeneration = bootstrapSubscriber.waitNextGeneration(); - log.log(DEBUG, "getConfigsOptional: new bootstrap generation: " + bootstrapSubscriber.generation()); + log.log(FINE, "getConfigsOptional: new bootstrap generation: " + bootstrapSubscriber.generation()); Optional<ConfigSnapshot> bootstrapConfig = bootstrapConfigIfChanged(); if (bootstrapConfig.isPresent()) { return bootstrapConfig; } else { if (newestBootstrapGeneration == newestComponentGeneration) { - log.log(DEBUG, "Got new components configs with unchanged bootstrap configs."); + log.log(FINE, "Got new components configs with unchanged bootstrap configs."); return componentsConfigIfChanged(); } else { // This should not be a normal case, and hence a warning to allow investigation. @@ -155,7 +155,7 @@ public final class ConfigRetriever { componentSubscriber.close(); componentSubscriberKeys = keys; try { - log.log(DEBUG, "Setting up new component subscriber for keys: " + keys); + log.log(FINE, "Setting up new component subscriber for keys: " + keys); componentSubscriber = subscribe.apply(keys); } catch (Throwable e) { log.log(Level.WARNING, "Failed setting up subscriptions for component configs: " + e.getMessage()); diff --git a/container-di/src/main/java/com/yahoo/container/di/Container.java b/container-di/src/main/java/com/yahoo/container/di/Container.java index 9a9245f4ba2..d8fec35633b 100644 --- a/container-di/src/main/java/com/yahoo/container/di/Container.java +++ b/container-di/src/main/java/com/yahoo/container/di/Container.java @@ -30,7 +30,7 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; /** * @author gjoranv @@ -139,7 +139,7 @@ public class Container { while (true) { snapshot = configurer.getConfigs(graph.configKeys(), leastGeneration, restartOnRedeploy); - log.log(DEBUG, String.format("createNewGraph:\n" + "graph.configKeys = %s\n" + "graph.generation = %s\n" + "snapshot = %s\n", + log.log(FINE, String.format("createNewGraph:\n" + "graph.configKeys = %s\n" + "graph.generation = %s\n" + "snapshot = %s\n", graph.configKeys(), graph.generation(), snapshot)); if (snapshot instanceof BootstrapConfigs) { @@ -148,7 +148,7 @@ public class Container { "Got bootstrap configs out of sequence for old config generation %d.\n" + "Previous config generation is %d", getBootstrapGeneration(), previousConfigGeneration)); } - log.log(DEBUG, + log.log(FINE, String.format( "Got new bootstrap generation\n" + "bootstrap generation = %d\n" + "components generation: %d\n" + "previous generation: %d\n", @@ -165,7 +165,7 @@ public class Container { break; } } - log.log(DEBUG, + log.log(FINE, String.format( "Got components configs,\n" + "bootstrap generation = %d\n" + "components generation: %d\n" + "previous generation: %d", diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java index d07aefef75b..fdaaf4b698d 100644 --- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java +++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentGraph.java @@ -15,7 +15,7 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.container.di.componentgraph.cycle.CycleFinder; import com.yahoo.container.di.componentgraph.cycle.Graph; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConfigKey; import java.lang.annotation.Annotation; @@ -255,7 +255,7 @@ public class ComponentGraph { if (component.isEmpty()) { Object instance; try { - log.log(LogLevel.DEBUG, "Trying the fallback injector to create" + messageForNoGlobalComponent(clazz, node)); + log.log(Level.FINE, "Trying the fallback injector to create" + messageForNoGlobalComponent(clazz, node)); instance = fallbackInjector.getInstance(key); } catch (ConfigurationException e) { throw removeStackTrace(new IllegalStateException( diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java index 6662ba85e5d..92695565d47 100644 --- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java +++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/ComponentNode.java @@ -29,7 +29,7 @@ import java.util.stream.Collectors; import static com.yahoo.container.di.componentgraph.core.Exceptions.cutStackTraceAtConstructor; import static com.yahoo.container.di.componentgraph.core.Exceptions.removeStackTrace; import static com.yahoo.container.di.componentgraph.core.Keys.createKey; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; /** * @author Tony Vaagenes @@ -148,9 +148,9 @@ public class ComponentNode extends Node { Object instance; try { - log.log(DEBUG, "Constructing " + idAndType()); + log.log(FINE, "Constructing " + idAndType()); instance = constructor.newInstance(actualArguments.toArray()); - log.log(DEBUG, "Finished constructing " + idAndType()); + log.log(FINE, "Finished constructing " + idAndType()); } catch (InvocationTargetException | InstantiationException | IllegalAccessException e) { StackTraceElement dependencyInjectorMarker = new StackTraceElement("============= Dependency Injection =============", "newInstance", null, -1); diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java index 9ed89a6091a..3afc8bb817c 100644 --- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java +++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/core/Node.java @@ -15,8 +15,7 @@ import java.util.Optional; import java.util.Set; import java.util.logging.Logger; -import static com.yahoo.log.LogLevel.DEBUG; -import static com.yahoo.log.LogLevel.SPAM; +import static java.util.logging.Level.FINE; /** * @author Tony Vaagenes diff --git a/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java b/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java index ed9b1350b1c..327949bb8d0 100644 --- a/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java +++ b/container-di/src/main/java/com/yahoo/container/di/componentgraph/cycle/CycleFinder.java @@ -11,7 +11,7 @@ import java.util.stream.Collectors; import static com.yahoo.container.di.componentgraph.cycle.CycleFinder.State.BLACK; import static com.yahoo.container.di.componentgraph.cycle.CycleFinder.State.GRAY; import static com.yahoo.container.di.componentgraph.cycle.CycleFinder.State.WHITE; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.Collections.singletonList; @@ -58,7 +58,7 @@ public class CycleFinder<T> { if (visitDepthFirst(vertex, new ArrayList<>(singletonList(vertex)))) { if (cycle == null) throw new IllegalStateException("Null cycle - this should never happen"); if (cycle.isEmpty()) throw new IllegalStateException("Empty cycle - this should never happen"); - log.log(DEBUG, "Cycle detected: " + cycle); + log.log(FINE, "Cycle detected: " + cycle); return cycle; } } @@ -68,7 +68,7 @@ public class CycleFinder<T> { private boolean visitDepthFirst(T vertex, List<T> path) { colors.put(vertex, GRAY); - log.log(DEBUG, "Vertex start " + vertex + " - colors: " + colors + " - path: " + path); + log.log(FINE, "Vertex start " + vertex + " - colors: " + colors + " - path: " + path); for (T adjacent : graph.getAdjacent(vertex)) { path.add(adjacent); if (colors.get(adjacent) == GRAY) { @@ -81,7 +81,7 @@ public class CycleFinder<T> { path.remove(adjacent); } colors.put(vertex, BLACK); - log.log(DEBUG, "Vertex end " + vertex + " - colors: " + colors + " - path: " + path); + log.log(FINE, "Vertex end " + vertex + " - colors: " + colors + " - path: " + path); return false; } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index e174de19394..2a72d0b3442 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -37,7 +37,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Transport; import com.yahoo.jrt.slobrok.api.Register; import com.yahoo.jrt.slobrok.api.SlobrokList; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.messagebus.network.rpc.SlobrokConfigSubscriber; import com.yahoo.net.HostName; @@ -109,7 +109,7 @@ public final class ConfiguredApplication implements Application { static { LogSetup.initVespaLogging("Container"); - log.log(LogLevel.INFO, "Starting container"); + log.log(Level.INFO, "Starting container"); } /** @@ -177,7 +177,7 @@ public final class ConfiguredApplication implements Application { Spec mySpec = new Spec(HostName.getLocalhost(), acceptor.port()); slobrokRegistrator = new Register(supervisor, slobrokList, mySpec); slobrokRegistrator.registerName(qrConfig.rpc().slobrokId()); - log.log(LogLevel.INFO, "Registered name '" + qrConfig.rpc().slobrokId() + + log.log(Level.INFO, "Registered name '" + qrConfig.rpc().slobrokId() + "' at " + mySpec + " with: " + slobrokList); return slobrokRegistrator; } @@ -210,7 +210,7 @@ public final class ConfiguredApplication implements Application { Container.get().setupFileAcquirer(config.filedistributor()); Container.get().setupUrlDownloader(); } catch (Exception e) { - log.log(LogLevel.ERROR, "Caught exception when initializing server. Exiting.", e); + log.log(Level.SEVERE, "Caught exception when initializing server. Exiting.", e); Runtime.getRuntime().halt(1); } } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java index 6a46e331762..e60e8d407cd 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/component/Deconstructor.java @@ -6,7 +6,7 @@ import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.container.di.ComponentDeconstructor; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.jdisc.SharedResource; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.osgi.framework.Bundle; import org.osgi.framework.BundleException; @@ -19,7 +19,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; import static java.util.logging.Level.SEVERE; import static java.util.logging.Level.WARNING; @@ -52,11 +52,11 @@ public class Deconstructor implements ComponentDeconstructor { } } else if (component instanceof Provider) { // TODO Providers should most likely be deconstructed similarly to AbstractComponent - log.log(DEBUG, () -> "Starting deconstruction of provider " + component); + log.log(FINE, () -> "Starting deconstruction of provider " + component); ((Provider<?>) component).deconstruct(); - log.log(DEBUG, () -> "Finished deconstruction of provider " + component); + log.log(FINE, () -> "Finished deconstruction of provider " + component); } else if (component instanceof SharedResource) { - log.log(DEBUG, () -> "Releasing container reference to resource " + component); + log.log(FINE, () -> "Releasing container reference to resource " + component); // No need to delay release, as jdisc does ref-counting ((SharedResource) component).release(); } @@ -89,16 +89,16 @@ public class Deconstructor implements ComponentDeconstructor { @Override public void run() { for (var component : components) { - log.log(DEBUG, () -> "Starting deconstruction of component " + component); + log.log(FINE, () -> "Starting deconstruction of component " + component); try { component.deconstruct(); - log.log(DEBUG, () -> "Finished deconstructing of component " + component); + log.log(FINE, () -> "Finished deconstructing of component " + component); } catch (Exception | NoClassDefFoundError e) { // May get class not found due to it being already unloaded log.log(WARNING, "Exception thrown when deconstructing component " + component, e); } catch (Error e) { try { Duration shutdownDelay = getRandomizedShutdownDelay(); - log.log(LogLevel.FATAL, "Error when deconstructing component " + component + ". Will sleep for " + + log.log(Level.SEVERE, "Error when deconstructing component " + component + ". Will sleep for " + shutdownDelay.getSeconds() + " seconds then restart", e); Thread.sleep(shutdownDelay.toMillis()); } catch (InterruptedException exception) { diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java index 1ab418fd523..a609ee73a97 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/MbusServerProvider.java @@ -5,7 +5,7 @@ import com.yahoo.component.ComponentId; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.jdisc.ReferencedResource; import com.yahoo.jdisc.service.CurrentContainer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.IntermediateSessionParams; import com.yahoo.messagebus.jdisc.MbusServer; import com.yahoo.messagebus.shared.SharedIntermediateSession; @@ -48,7 +48,7 @@ public class MbusServerProvider implements Provider<MbusServer> { @Override public void deconstruct() { - log.log(LogLevel.INFO, "Deconstructing mbus server: " + server); + log.log(Level.INFO, "Deconstructing mbus server: " + server); server.close(); server.release(); sessionRef.getReference().close(); diff --git a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java index 7193433ccf7..e62f6a8a21a 100644 --- a/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java +++ b/container-messagebus/src/main/java/com/yahoo/container/jdisc/messagebus/SessionCache.java @@ -12,7 +12,7 @@ import com.yahoo.jdisc.ReferencedResource; import com.yahoo.jdisc.References; import com.yahoo.jdisc.ResourceReference; import com.yahoo.jdisc.SharedResource; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.ConfigAgent; import com.yahoo.messagebus.DynamicThrottlePolicy; import com.yahoo.messagebus.IntermediateSessionParams; @@ -128,16 +128,16 @@ public final class SessionCache extends AbstractComponent { } private static void logSystemInfo(ContainerMbusConfig containerMbusConfig, long maxPendingSize) { - log.log(LogLevel.DEBUG, + log.log(Level.FINE, "Running with maximum heap size of " + (Runtime.getRuntime().maxMemory() / 1024L / 1024L) + " MB"); - log.log(LogLevel.CONFIG, + log.log(Level.CONFIG, "Amount of memory reserved for container core: " + containerMbusConfig.containerCoreMemory() + " MB."); - log.log(LogLevel.CONFIG, + log.log(Level.CONFIG, "Running with document expansion factor " + containerMbusConfig.documentExpansionFactor() + ""); String msgLimit = (containerMbusConfig.maxpendingcount() == 0) ? "unlimited" : "" + containerMbusConfig.maxpendingcount(); - log.log(LogLevel.CONFIG, ("Starting message bus with max " + msgLimit + " pending messages and max " + + log.log(Level.CONFIG, ("Starting message bus with max " + msgLimit + " pending messages and max " + (((double) (maxPendingSize / 1024L)) / 1024.0d) + " pending megabytes.")); } @@ -204,7 +204,7 @@ public final class SessionCache extends AbstractComponent { @Override SharedSourceSession create(SourceSessionParams p) { - log.log(LogLevel.DEBUG, "Creating new source session."); + log.log(Level.FINE, "Creating new source session."); return messageBus.newSourceSession(p); } @@ -215,7 +215,7 @@ public final class SessionCache extends AbstractComponent { @Override void logReuse(final SharedSourceSession session) { - log.log(LogLevel.DEBUG, "Reusing source session."); + log.log(Level.FINE, "Reusing source session."); } } @@ -224,7 +224,7 @@ public final class SessionCache extends AbstractComponent { @Override SharedIntermediateSession create(IntermediateSessionParams p) { - log.log(LogLevel.DEBUG, "Creating new intermediate session " + p.getName() + ""); + log.log(Level.FINE, "Creating new intermediate session " + p.getName() + ""); return messageBus.newIntermediateSession(p); } @@ -235,7 +235,7 @@ public final class SessionCache extends AbstractComponent { @Override void logReuse(SharedIntermediateSession session) { - log.log(LogLevel.DEBUG, "Reusing intermediate session " + session.name() + ""); + log.log(Level.FINE, "Reusing intermediate session " + session.name() + ""); } } diff --git a/container-search-and-docproc/pom.xml b/container-search-and-docproc/pom.xml index 91202087df2..cc66bd66922 100644 --- a/container-search-and-docproc/pom.xml +++ b/container-search-and-docproc/pom.xml @@ -148,12 +148,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>documentapi</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java index 329a9caaf91..ef892585d21 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/DocsumField.java @@ -2,7 +2,7 @@ package com.yahoo.prelude.fastsearch; import com.yahoo.data.access.Inspector; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; @@ -58,7 +58,7 @@ public abstract class DocsumField { fieldFactory.put("xmlstring", XMLField.class); fieldFactory.put("tensor", TensorField.class); } catch (Exception e) { - log.log(LogLevel.ERROR, "Could not initialize docsum decoding properly.", e); + log.log(Level.SEVERE, "Could not initialize docsum decoding properly.", e); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java index 8020088c2e3..95a374316de 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SpecialTokens.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.query.Substring; import java.util.*; @@ -66,7 +66,7 @@ public class SpecialTokens { // TODO put along with the global toLowerCase String asHigh = token.toUpperCase(Locale.ENGLISH); if (asLow.length() != token.length() || asHigh.length() != token.length()) { - log.log(LogLevel.ERROR, "Special token '" + token + "' has case sensitive length. Ignoring the token." + log.log(Level.SEVERE, "Special token '" + token + "' has case sensitive length. Ignoring the token." + " Please report this message in a bug to the Vespa team."); return false; } else { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index 9a9044def2d..318912eab04 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -9,7 +9,7 @@ import com.yahoo.language.Language; import com.yahoo.language.Linguistics; import com.yahoo.language.process.StemMode; import com.yahoo.language.process.StemList; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.*; @@ -98,6 +98,7 @@ public class StemmingSearcher extends Searcher { context.language = language; context.indexFacts = indexFacts; context.reverseConnectivity = createReverseConnectivities(q.getModel().getQueryTree().getRoot()); + q.trace("Stemming with language="+language, 3); return scan(q.getModel().getQueryTree().getRoot(), context); } @@ -167,7 +168,7 @@ public class StemmingSearcher extends Searcher { if (i instanceof TermItem) { return ((TermItem) i).getOrigin(); // this should always be the case } else { - getLogger().log(LogLevel.WARNING, "Weird, BlockItem '" + b + "' was a composite containing " + + getLogger().log(Level.WARNING, "Weird, BlockItem '" + b + "' was a composite containing " + i.getClass().getName() + ", expected TermItem."); } } @@ -183,9 +184,20 @@ public class StemmingSearcher extends Searcher { Substring substring = getOffsets(current); if (segments.size() == 1) { + getLogger().log(Level.FINE, () -> "Stem '"+current.stringValue()+"' mode "+index.getStemMode() + +" and language '"+context.language+"' -> '"+segments.get(0)+"'"); TaggableItem w = singleWordSegment(current, segments.get(0), index, substring, context.insidePhrase); setMetaData(current, context.reverseConnectivity, w); return (Item) w; + } else if (getLogger().isLoggable(Level.FINE)) { + var buf = new StringBuilder(); + buf.append("Stem '").append(current.stringValue()); + buf.append("' mode ").append(index.getStemMode()); + buf.append(" and language '").append(context.language).append("' ->"); + for (StemList segment : segments) { + buf.append(" '").append(segment).append("'"); + } + getLogger().log(Level.FINE, buf.toString()); } if (context.isCJK) @@ -194,6 +206,7 @@ public class StemmingSearcher extends Searcher { composite = chooseComposite(current, ((Item) current).getParent(), indexName); for (StemList segment : segments) { + getLogger().log(Level.FINE, () -> "Stem to multiple segments '"+segment+"'"); TaggableItem w = singleWordSegment(current, segment, index, substring, context.insidePhrase); if (composite instanceof AndSegmentItem) { diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index ebea2f95af7..b4b23a0e94a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -5,7 +5,7 @@ import com.yahoo.component.chain.dependencies.Before; import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.container.protect.Error; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.metrics.simple.MetricSettings; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.processing.request.CompoundName; @@ -243,7 +243,7 @@ public class StatisticsSearcher extends Searcher { if (latency >= 0) { addLatency(latency, metricContext); } else { - getLogger().log(LogLevel.WARNING, + getLogger().log(Level.WARNING, "Apparently negative latency measure, start: " + start + ", end: " + end + ", for query: " + query.toString()); } diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index dc8e2b70740..9e3f6d20e36 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -7,7 +7,7 @@ import com.yahoo.collections.Tuple2; import com.yahoo.component.Version; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.fs4.MapEncoder; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.fastsearch.DocumentDatabase; import com.yahoo.prelude.query.Highlight; import com.yahoo.prelude.query.QueryException; @@ -726,7 +726,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { if (includeQuery) message += ": [" + queryTreeText() + "]"; - log.log(LogLevel.DEBUG,message); + log.log(Level.FINE,message); // Pass 0 as traceLevel as the trace level check is already done above, // and it is not propagated to trace until execution has started diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 645c6446ef1..db79c610dc7 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -3,7 +3,7 @@ package com.yahoo.search.cluster; import com.yahoo.component.ComponentId; import com.yahoo.container.protect.Error; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; import com.yahoo.yolean.Exceptions; @@ -71,7 +71,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod /** Pinging a node, called from ClusterMonitor */ @Override public final void ping(ClusterMonitor<T> clusterMonitor, T p, Executor executor) { - log(LogLevel.FINE, "Sending ping to: ", p); + log(Level.FINE, "Sending ping to: ", p); Pinger pinger = new Pinger(p); FutureTask<Pong> future = new FutureTask<>(pinger); @@ -97,10 +97,10 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod if (pong.badResponse()) { clusterMonitor.failed(p, pong.error().get()); - log(LogLevel.FINE, "Failed ping - ", pong); + log(Level.FINE, "Failed ping - ", pong); } else { clusterMonitor.responded(p); - log(LogLevel.FINE, "Answered ping - ", p); + log(Level.FINE, "Answered ping - ", p); } if (logThrowable != null) { @@ -173,7 +173,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod if (result.hits().getError().getCode() == Error.TIMEOUT.code) return result; // Retry is unlikely to help - log(LogLevel.FINER, "No result, checking for timeout."); + log(Level.FINER, "No result, checking for timeout."); tries++; connection = nodes.select(code, tries); } while (tries < nodes.getNodeCount()); @@ -211,7 +211,7 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod try { result = search(query, execution, connection); } catch (RuntimeException e) { //TODO: Exceptions should not be used to signal backend communication errors - log(LogLevel.WARNING, "An exception occurred while invoking backend searcher.", e); + log(Level.WARNING, "An exception occurred while invoking backend searcher.", e); result = new Result(query, ErrorMessage.createBackendCommunicationError("Failed calling " + connection + " in " + this + " for " + query + ": " + Exceptions.toMessageString(e))); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java index 8280917ca2f..3abfa3a6531 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java @@ -3,7 +3,7 @@ package com.yahoo.search.grouping; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -110,7 +110,7 @@ public class UniqueGroupingSearcher extends Searcher { if (null == root) { String msg = "Result group not found for deduping grouping request, returning empty result."; query.trace(msg, 3); - log.log(LogLevel.WARNING, msg); + log.log(Level.WARNING, msg); throw new IllegalStateException("Failed to produce deduped result set."); } result.hits().remove(root.getId().toString()); // hide our tracks diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index 7c2e774f68b..d2d89e41879 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -13,7 +13,7 @@ import java.util.logging.Logger; import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Provides; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.QueryCanonicalizer; @@ -225,9 +225,9 @@ public class GroupingExecutor extends Searcher { if (passList.isEmpty()) { throw new RuntimeException("No grouping request for pass " + pass + ", bug!"); } - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { for (Grouping grouping : passList) { - log.log(LogLevel.DEBUG, "Pass(" + pass + "), Grouping(" + grouping.getId() + "): " + grouping); + log.log(Level.FINE, "Pass(" + pass + "), Grouping(" + grouping.getId() + "): " + grouping); } } Item passRoot; @@ -263,9 +263,9 @@ public class GroupingExecutor extends Searcher { ret = passResult; } } - if (log.isLoggable(LogLevel.DEBUG)) { + if (log.isLoggable(Level.FINE)) { for (Grouping grouping : groupingMap.values()) { - log.log(LogLevel.DEBUG, "Result Grouping(" + grouping.getId() + "): " + grouping); + log.log(Level.FINE, "Result Grouping(" + grouping.getId() + "): " + grouping); } } return ret; diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 5e3b79c1545..c8386f3c75c 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -18,7 +18,7 @@ import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; import com.yahoo.jdisc.Metric; import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.net.HostName; import com.yahoo.net.UriTools; import com.yahoo.prelude.query.QueryException; @@ -154,7 +154,7 @@ public class SearchHandler extends LoggingRequestHandler { long numQueriesToTraceOnDebugAfterStartup, Optional<String> hostResponseHeaderKey) { super(executor, accessLog, metric, true); - log.log(LogLevel.DEBUG, "SearchHandler.init " + System.identityHashCode(this)); + log.log(Level.FINE, "SearchHandler.init " + System.identityHashCode(this)); this.queryProfileRegistry = queryProfileRegistry; this.executionFactory = executionFactory; @@ -417,7 +417,7 @@ public class SearchHandler extends LoggingRequestHandler { if (searchConnections != null) { connectionStatistics(); } else { - log.log(LogLevel.WARNING, + log.log(Level.WARNING, "searchConnections is a null reference, probably a known race condition during startup.", new IllegalStateException("searchConnections reference is null.")); } @@ -426,7 +426,7 @@ public class SearchHandler extends LoggingRequestHandler { } catch (ParseException e) { ErrorMessage error = ErrorMessage.createIllegalQuery("Could not parse query [" + request + "]: " + Exceptions.toMessageString(e)); - log.log(LogLevel.DEBUG, error::getDetailedMessage); + log.log(Level.FINE, error::getDetailedMessage); return new Result(query, error); } catch (IllegalArgumentException e) { if ("Comparison method violates its general contract!".equals(e.getMessage())) { @@ -438,7 +438,7 @@ public class SearchHandler extends LoggingRequestHandler { else { ErrorMessage error = ErrorMessage.createBadRequest("Invalid search request [" + request + "]: " + Exceptions.toMessageString(e)); - log.log(LogLevel.DEBUG, error::getDetailedMessage); + log.log(Level.FINE, error::getDetailedMessage); return new Result(query, error); } } catch (LinkageError | StackOverflowError e) { @@ -485,10 +485,10 @@ public class SearchHandler extends LoggingRequestHandler { private void log(String request, Query query, Throwable e) { // Attempted workaround for missing stack traces if (e.getStackTrace().length == 0) { - log.log(LogLevel.ERROR, "Failed executing " + query.toDetailString() + + log.log(Level.SEVERE, "Failed executing " + query.toDetailString() + " [" + request + "], received exception with no context", e); } else { - log.log(LogLevel.ERROR, "Failed executing " + query.toDetailString() + " [" + request + "]", e); + log.log(Level.SEVERE, "Failed executing " + query.toDetailString() + " [" + request + "]", e); } } diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java index b73ef1298ee..d66aff7bcd7 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterUtils.java @@ -2,7 +2,7 @@ package com.yahoo.search.query.rewrite; import com.yahoo.fsa.FSA; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.search.Query; import com.yahoo.search.intent.model.IntentModel; import com.yahoo.search.intent.model.InterpretationNode; @@ -289,7 +289,7 @@ public class RewriterUtils { * @param msg Log message */ public static void log(Logger logger, String msg) { - logger.log(LogLevel.DEBUG, logger.getName() + ": " + msg); + logger.log(Level.FINE, logger.getName() + ": " + msg); } /** @@ -303,7 +303,7 @@ public class RewriterUtils { if(query!=null) { query.trace(logger.getName() + ": " + msg, true, TRACELEVEL); } - logger.log(LogLevel.DEBUG, logger.getName() + ": " + msg); + logger.log(Level.FINE, logger.getName() + ": " + msg); } /** diff --git a/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java index 9d0e110a6dd..d79122ac9c4 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java @@ -2,7 +2,7 @@ package com.yahoo.search.rendering; import com.yahoo.concurrent.CopyOnWriteHashMap; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.net.URI; import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.prelude.hitfield.HitField; @@ -395,7 +395,7 @@ public final class SyncDefaultRenderer extends Renderer { if (e instanceof IOException) { throw (IOException) e; } else { - log.log(LogLevel.WARNING, "Exception thrown when rendering the result:", e); + log.log(Level.WARNING, "Exception thrown when rendering the result:", e); } } diff --git a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java index 018f4fc7aa9..3c0453f8900 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/InputCheckingSearcher.java @@ -11,7 +11,7 @@ import java.util.ListIterator; import java.util.Map; import java.util.logging.Logger; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.metrics.simple.Counter; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.prelude.query.CompositeItem; @@ -51,8 +51,8 @@ public class InputCheckingSearcher extends Searcher { try { checkQuery(query); } catch (IllegalArgumentException e) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Rejected query \"" + query.toString() + "\" on cause of: " + e.getMessage()); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Rejected query \"" + query.toString() + "\" on cause of: " + e.getMessage()); } return new Result(query, ErrorMessage.createIllegalQuery(e.getMessage())); } diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java index 3c96b00d628..b9e19d6a1b6 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/TracingOptions.java @@ -15,12 +15,12 @@ import java.util.concurrent.TimeUnit; * Encapsulates all trace-related components and options used by the streaming search Searcher. * * Provides a DEFAULT static instance which has the following characteristics: - * - Approximately 1 query every 2 seconds is traced + * - Approximately 1 query every second is traced * - Trace level is set to 7 for traced queries - * - Only emits traces for queries that have timed out and where the elapsed time is at least 5x + * - Only emits traces for queries that have timed out and where the elapsed time is at least 2x * of the timeout specified in the query itself * - Emits traces to the Vespa log - * - Only 1 trace every 10 seconds may be emitted to the log + * - Only 2 traces every 10 seconds may be emitted to the log */ public class TracingOptions { @@ -50,11 +50,11 @@ public class TracingOptions { public static final TracingOptions DEFAULT; public static final int DEFAULT_TRACE_LEVEL_OVERRIDE = 7; // TODO determine appropriate trace level // Traces won't be exported unless the query has timed out with a duration that is > timeout * multiplier - public static final double TRACE_TIMEOUT_MULTIPLIER_THRESHOLD = 5.0; + public static final double TRACE_TIMEOUT_MULTIPLIER_THRESHOLD = 2.0; static { - SamplingStrategy queryTraceSampler = ProbabilisticSampleRate.withSystemDefaults(0.5); - SamplingStrategy logExportSampler = MaxSamplesPerPeriod.withSteadyClock(TimeUnit.SECONDS.toNanos(10), 1); + SamplingStrategy queryTraceSampler = ProbabilisticSampleRate.withSystemDefaults(1); + SamplingStrategy logExportSampler = MaxSamplesPerPeriod.withSteadyClock(TimeUnit.SECONDS.toNanos(10), 2); TraceExporter traceExporter = new SamplingTraceExporter(new LoggingTraceExporter(), logExportSampler); DEFAULT = new TracingOptions(queryTraceSampler, traceExporter, System::nanoTime, DEFAULT_TRACE_LEVEL_OVERRIDE, TRACE_TIMEOUT_MULTIPLIER_THRESHOLD); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index 4750bac551c..8bd7bdac8a5 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -5,7 +5,7 @@ import com.yahoo.document.DocumentId; import com.yahoo.document.select.parser.ParseException; import com.yahoo.document.select.parser.TokenMgrException; import com.yahoo.fs4.DocsumPacket; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.routing.Route; import com.yahoo.prelude.Ping; import com.yahoo.prelude.Pong; @@ -300,9 +300,9 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { String expUserId = query.properties().getString(streamingUserid); String expGroupName = query.properties().getString(streamingGroupname); - LogLevel logLevel = LogLevel.ERROR; + Level logLevel = Level.SEVERE; if (skippedEarlierResult) { - logLevel = LogLevel.DEBUG; + logLevel = Level.FINE; } DocumentId docId; diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index 795b62663d5..2b0d8adc8ef 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -17,7 +17,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentSummaryMessage; import com.yahoo.documentapi.messagebus.protocol.QueryResultMessage; import com.yahoo.documentapi.messagebus.protocol.SearchResultMessage; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Trace; import com.yahoo.messagebus.routing.Route; @@ -142,9 +142,9 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private int inferSessionTraceLevel(Query query) { int implicitLevel = traceLevelOverride; - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { implicitLevel = 9; - } else if (log.isLoggable(LogLevel.DEBUG)) { + } else if (log.isLoggable(Level.FINE)) { implicitLevel = 7; } return Math.max(query.getTraceLevel(), implicitLevel); @@ -330,20 +330,20 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { VisitorSession session = visitorSessionFactory.createVisitorSession(params); try { if ( !session.waitUntilDone(query.getTimeout())) { - log.log(LogLevel.DEBUG, "Visitor returned from waitUntilDone without being completed for " + query + " with selection " + params.getDocumentSelection()); + log.log(Level.FINE, "Visitor returned from waitUntilDone without being completed for " + query + " with selection " + params.getDocumentSelection()); session.abort(); throw new TimeoutException("Query timed out in " + VdsStreamingSearcher.class.getName()); } } finally { session.destroy(); sessionTrace = session.getTrace(); - log.log(LogLevel.DEBUG, () -> sessionTrace.toString()); + log.log(Level.FINE, () -> sessionTrace.toString()); query.trace(sessionTrace.toString(), false, 9); } if (params.getControlHandler().getResult().code == VisitorControlHandler.CompletionCode.SUCCESS) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "VdsVisitor completed successfully for " + query + " with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "VdsVisitor completed successfully for " + query + " with selection " + params.getDocumentSelection()); } } else { throw new IllegalArgumentException("Query failed: " // TODO: Is it necessary to use a runtime exception? @@ -384,8 +384,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } public void onSearchResult(SearchResult sr) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Got SearchResult for query with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Got SearchResult for query with selection " + params.getDocumentSelection()); } handleSearchResult(sr); } @@ -393,8 +393,8 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { private void handleSearchResult(SearchResult sr) { final int hitCountTotal = sr.getTotalHitCount(); final int hitCount = sr.getHitCount(); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Got SearchResult with " + hitCountTotal + " in total and " + hitCount + " hits in real for query with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Got SearchResult with " + hitCountTotal + " in total and " + hitCount + " hits in real for query with selection " + params.getDocumentSelection()); } List<SearchResult.Hit> newHits = new ArrayList<>(hitCount); @@ -412,15 +412,15 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } private void mergeGroupingMaps(Map<Integer, byte []> newGroupingMap) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "mergeGroupingMaps: newGroupingMap = " + newGroupingMap); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "mergeGroupingMaps: newGroupingMap = " + newGroupingMap); } for(Integer key : newGroupingMap.keySet()) { byte [] value = newGroupingMap.get(key); Grouping newGrouping = new Grouping(); - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Received group with key " + key + " and size " + value.length); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Received group with key " + key + " and size " + value.length); } BufferSerializer buf = new BufferSerializer( new GrowableByteBuffer(ByteBuffer.wrap(value)) ); newGrouping.deserialize(buf); @@ -440,16 +440,16 @@ class VdsVisitor extends VisitorDataHandler implements Visitor { } public void onDocumentSummary(DocumentSummary ds) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Got DocumentSummary for query with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Got DocumentSummary for query with selection " + params.getDocumentSelection()); } handleSummary(ds); } private void handleSummary(DocumentSummary ds) { int summaryCount = ds.getSummaryCount(); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Got DocumentSummary with " + summaryCount + " summaries for query with selection " + params.getDocumentSelection()); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Got DocumentSummary with " + summaryCount + " summaries for query with selection " + params.getDocumentSelection()); } synchronized (summaryMap) { for (int i = 0; i < summaryCount; i++) { diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java index 0aaf301e071..230af8971c2 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/tracing/LoggingTraceExporter.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.streamingvisitors.tracing; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.function.Supplier; import java.util.logging.Logger; @@ -17,7 +17,7 @@ public class LoggingTraceExporter implements TraceExporter { public void maybeExport(Supplier<TraceDescription> traceDescriptionSupplier) { var traceDescription = traceDescriptionSupplier.get(); if (traceDescription.getTrace() != null) { - log.log(LogLevel.WARNING, String.format("%s: %s", traceDescription.getDescription(), + log.log(Level.WARNING, String.format("%s: %s", traceDescription.getDescription(), traceDescription.getTrace().toString())); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java index 384b1342ea4..02c6972b543 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/LogEntry.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.controller.api.integration; import com.yahoo.log.LogLevel; +import java.util.logging.Level; + import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -12,7 +14,6 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Objects; -import java.util.logging.Level; import java.util.stream.Collectors; import static java.nio.charset.StandardCharsets.UTF_8; @@ -103,9 +104,9 @@ public class LogEntry { } public static Type typeOf(Level level) { - return level.intValue() < LogLevel.INFO.intValue() ? Type.debug - : level.intValue() < LogLevel.WARNING.intValue() ? Type.info - : level.intValue() < LogLevel.ERROR.intValue() ? Type.warning + return level.intValue() < Level.INFO.intValue() ? Type.debug + : level.intValue() < Level.WARNING.intValue() ? Type.info + : level.intValue() < Level.SEVERE.intValue() ? Type.warning : Type.error; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 05e9707473c..c5a016d53c8 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -49,7 +49,7 @@ public interface NodeRepository { .collect(Collectors.toUnmodifiableList()); } - /** List all nodes in zone owned by given application */ + /** List all nodes in zone having given hostnames */ default List<Node> list(ZoneId zone, List<HostName> hostnames) { return listNodes(zone, hostnames).nodes().stream() .map(NodeRepository::toNode) diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 5c11dfc2a55..ced3d201f6f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -58,13 +58,25 @@ enum PathGroup { "/application/v4/tenant/{tenant}/application/", "/application/v4/tenant/{tenant}/cost", "/application/v4/tenant/{tenant}/cost/{date}", - "/routing/v1/status/tenant/{tenant}/{*}", - "/billing/v1/tenant/{tenant}/{*}"), + "/routing/v1/status/tenant/{tenant}/{*}"), tenantKeys(Matcher.tenant, PathPrefix.api, "/application/v4/tenant/{tenant}/key/"), + + billingToken(Matcher.tenant, + PathPrefix.api, + "/billing/v1/tenant/{tenant}/token"), + + billingInstrument(Matcher.tenant, + PathPrefix.api, + "/billing/v1/tenant/{tenant}/instrument/{*}"), + + billingList(Matcher.tenant, + PathPrefix.api, + "/billing/v1/tenant/{tenant}/billing/{*}"), + applicationKeys(Matcher.tenant, Matcher.application, PathPrefix.api, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java index cfe8d247e54..0afa0668a00 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Policy.java @@ -142,7 +142,32 @@ enum Policy { /** Access to /payment/notification */ paymentProcessor(Privilege.grant(Action.create) .on(PathGroup.paymentProcessor) - .in(SystemName.PublicCd)); + .in(SystemName.PublicCd)), + + /** Read your own instrument information */ + paymentInstrumentRead(Privilege.grant(Action.read) + .on(PathGroup.billingInstrument) + .in(SystemName.PublicCd)), + + /** Ability to update tenant payment instrument */ + paymentInstrumentUpdate(Privilege.grant(Action.update) + .on(PathGroup.billingInstrument) + .in(SystemName.PublicCd)), + + /** Ability to remove your own payment instrument */ + paymentInstrumentDelete(Privilege.grant(Action.delete) + .on(PathGroup.billingInstrument) + .in(SystemName.PublicCd)), + + /** Get the token to view instrument form */ + paymentInstrumentCreate(Privilege.grant(Action.read) + .on(PathGroup.billingToken) + .in(SystemName.PublicCd)), + + /** Read the generated bills */ + billingInformationRead(Privilege.grant(Action.read) + .on(PathGroup.billingList) + .in(SystemName.PublicCd)); private final Set<Privilege> privileges; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java index c05936ee593..438e79bcc4f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/RoleDefinition.java @@ -43,7 +43,10 @@ public enum RoleDefinition { reader(Policy.tenantRead, Policy.applicationRead, Policy.deploymentRead, - Policy.publicRead), + Policy.publicRead, + Policy.paymentInstrumentRead, + Policy.paymentInstrumentDelete, + Policy.billingInformationRead), /** User — the dev.ops. role for normal Vespa tenant users */ developer(Policy.applicationCreate, @@ -52,12 +55,20 @@ public enum RoleDefinition { Policy.applicationOperations, Policy.developmentDeployment, Policy.keyManagement, - Policy.submission), + Policy.submission, + Policy.paymentInstrumentRead, + Policy.paymentInstrumentDelete, + Policy.billingInformationRead), /** Admin — the administrative function for user management etc. */ administrator(Policy.tenantUpdate, Policy.tenantManager, - Policy.applicationManager), + Policy.applicationManager, + Policy.paymentInstrumentRead, + Policy.paymentInstrumentUpdate, + Policy.paymentInstrumentDelete, + Policy.paymentInstrumentCreate, + Policy.billingInformationRead), /** Headless — the application specific role identified by deployment keys for production */ headless(Policy.submission), diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index 57b4af9d16c..2da93c5ceca 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -6,8 +6,10 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import org.junit.Test; +import java.awt.event.AdjustmentEvent; import java.net.URI; import java.util.List; +import java.util.stream.Stream; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -19,6 +21,7 @@ public class RoleTest { private static final Enforcer mainEnforcer = new Enforcer(SystemName.main); private static final Enforcer publicEnforcer = new Enforcer(SystemName.Public); + private static final Enforcer publicCdEnforcer = new Enforcer(SystemName.PublicCd); @Test public void operator_membership() { @@ -143,4 +146,46 @@ public class RoleTest { } } + @Test + public void payment_instrument() { + URI paymentInstrumentUri = URI.create("/billing/v1/tenant/t1/instrument/foobar"); + URI tenantPaymentInstrumentUri = URI.create("/billing/v1/tenant/t1/instrument"); + URI tokenUri = URI.create("/billing/v1/tenant/t1/token"); + + Role user = Role.reader(TenantName.from("t1")); + assertTrue(publicCdEnforcer.allows(user, Action.read, paymentInstrumentUri)); + assertTrue(publicCdEnforcer.allows(user, Action.delete, paymentInstrumentUri)); + assertFalse(publicCdEnforcer.allows(user, Action.update, tenantPaymentInstrumentUri)); + assertFalse(publicCdEnforcer.allows(user, Action.read, tokenUri)); + + Role developer = Role.developer(TenantName.from("t1")); + assertTrue(publicCdEnforcer.allows(developer, Action.read, paymentInstrumentUri)); + assertTrue(publicCdEnforcer.allows(developer, Action.delete, paymentInstrumentUri)); + assertFalse(publicCdEnforcer.allows(developer, Action.update, tenantPaymentInstrumentUri)); + assertFalse(publicCdEnforcer.allows(developer, Action.read, tokenUri)); + + Role admin = Role.administrator(TenantName.from("t1")); + assertTrue(publicCdEnforcer.allows(admin, Action.read, paymentInstrumentUri)); + assertTrue(publicCdEnforcer.allows(admin, Action.delete, paymentInstrumentUri)); + assertTrue(publicCdEnforcer.allows(admin, Action.update, tenantPaymentInstrumentUri)); + assertTrue(publicCdEnforcer.allows(admin, Action.read, tokenUri)); + } + + @Test + public void billing() { + URI billing = URI.create("/billing/v1/tenant/t1/billing"); + + Role user = Role.reader(TenantName.from("t1")); + Role developer = Role.developer(TenantName.from("t1")); + Role admin = Role.administrator(TenantName.from("t1")); + + Stream.of(user, developer, admin).forEach(role -> { + assertTrue(publicCdEnforcer.allows(role, Action.read, billing)); + assertFalse(publicCdEnforcer.allows(role, Action.update, billing)); + assertFalse(publicCdEnforcer.allows(role, Action.delete, billing)); + assertFalse(publicCdEnforcer.allows(role, Action.create, billing)); + }); + + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index c3a9a8484c9..ae9d35e6790 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -6,7 +6,7 @@ import com.google.common.cache.CacheLoader; import com.google.inject.Inject; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; @@ -259,7 +259,7 @@ public class AthenzFacade implements AccessControl { } public List<AthenzDomain> getDomainList(String prefix) { - log.log(LogLevel.DEBUG, String.format("getDomainList(prefix=%s)", prefix)); + log.log(Level.FINE, String.format("getDomainList(prefix=%s)", prefix)); return zmsClient.getDomainList(prefix); } @@ -279,7 +279,7 @@ public class AthenzFacade implements AccessControl { } private static void log(String format, Object... args) { - log.log(LogLevel.DEBUG, String.format(format, args)); + log.log(Level.FINE, String.format(format, args)); } private String resourceStringPrefix(AthenzDomain tenantDomain) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index 4cb62892555..7e0fec4ba66 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.SubjectAlternativeName; import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.flags.BooleanFlag; @@ -86,7 +86,7 @@ public class EndpointCertificateManager { try { this.backfillCertificateMetadata(); } catch (Throwable t) { - log.log(LogLevel.INFO, "Unexpected Throwable caught while backfilling certificate metadata", t); + log.log(Level.INFO, "Unexpected Throwable caught while backfilling certificate metadata", t); } }, 1, 10, TimeUnit.MINUTES); } @@ -95,7 +95,7 @@ public class EndpointCertificateManager { var t0 = Instant.now(); Optional<EndpointCertificateMetadata> metadata = getOrProvision(instance, zone); Duration duration = Duration.between(t0, Instant.now()); - if (duration.toSeconds() > 30) log.log(LogLevel.INFO, String.format("Getting endpoint certificate metadata for %s took %d seconds!", instance.id().serializedForm(), duration.toSeconds())); + if (duration.toSeconds() > 30) log.log(Level.INFO, String.format("Getting endpoint certificate metadata for %s took %d seconds!", instance.id().serializedForm(), duration.toSeconds())); return metadata; } @@ -175,7 +175,7 @@ public class EndpointCertificateManager { EndpointCertificateMetadata providerMetadata = sanToEndpointCertificate.get(hashedCn); if (providerMetadata == null) { - log.log(LogLevel.INFO, "No matching certificate provider metadata found for application " + applicationId.serializedForm()); + log.log(Level.INFO, "No matching certificate provider metadata found for application " + applicationId.serializedForm()); return; } @@ -189,7 +189,7 @@ public class EndpointCertificateManager { providerMetadata.issuer()); if (mode == BackfillMode.DRYRUN) { - log.log(LogLevel.INFO, "Would update stored metadata " + storedMetaData + " with data from provider: " + backfilledMetadata); + log.log(Level.INFO, "Would update stored metadata " + storedMetaData + " with data from provider: " + backfilledMetadata); } else if (mode == BackfillMode.ENABLE) { curator.writeEndpointCertificateMetadata(applicationId, backfilledMetadata); } @@ -251,10 +251,10 @@ public class EndpointCertificateManager { // Normally because the cert is in the process of being provisioned - this will cause a retry in InternalStepRunner throw new EndpointCertificateException(EndpointCertificateException.Type.CERT_NOT_AVAILABLE, "Certificate not found in secret store"); } catch (EndpointCertificateException e) { - log.log(LogLevel.WARNING, "Certificate validation failure for " + instance.id().serializedForm(), e); + log.log(Level.WARNING, "Certificate validation failure for " + instance.id().serializedForm(), e); throw e; } catch (Exception e) { - log.log(LogLevel.WARNING, "Certificate validation failure for " + instance.id().serializedForm(), e); + log.log(Level.WARNING, "Certificate validation failure for " + instance.id().serializedForm(), e); throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Certificate validation failure for app " + instance.id().serializedForm(), e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 893956e5767..e2fb74ec0b4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -5,7 +5,7 @@ import com.yahoo.config.application.api.DeploymentInstanceSpec; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -69,7 +69,7 @@ public class DeploymentTrigger { public void notifyOfSubmission(TenantAndApplicationId id, ApplicationVersion version, long projectId) { if (applications().getApplication(id).isEmpty()) { - log.log(LogLevel.WARNING, "Ignoring submission from project '" + projectId + + log.log(Level.WARNING, "Ignoring submission from project '" + projectId + "': Unknown application '" + id + "'"); return; } @@ -117,7 +117,7 @@ public class DeploymentTrigger { */ public void notifyOfCompletion(ApplicationId id) { if (applications().getInstance(id).isEmpty()) { - log.log(LogLevel.WARNING, "Ignoring completion of job of unknown application '" + id + "'"); + log.log(Level.WARNING, "Ignoring completion of job of unknown application '" + id + "'"); return; } @@ -155,7 +155,7 @@ public class DeploymentTrigger { /** Attempts to trigger the given job. */ public void trigger(Job job) { - log.log(LogLevel.DEBUG, "Triggering " + job); + log.log(Level.FINE, "Triggering " + job); applications().lockApplicationOrThrow(TenantAndApplicationId.from(job.applicationId()), application -> { jobs.start(job.applicationId(), job.jobType, job.versions); applications().store(application.with(job.applicationId().instance(), instance -> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 1f2d503326a..bc206011bb1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -526,8 +526,8 @@ public class InternalStepRunner implements StepRunner { private Stream<String> nodeDetails(NodeWithServices node, boolean printAllServices) { return Stream.concat(Stream.of(node.node().hostname() + ": " + humanize(node.node().serviceState()) + (node.node().suspendedSince().map(since -> " since " + since).orElse("")), - "--- platform " + node.node().wantedVersion() + (node.needsPlatformUpgrade() - ? " <-- " + (node.node().currentVersion().isEmpty() ? "not booted" : node.node().currentVersion()) + "--- platform " + wantedPlatform(node.node()) + (node.needsPlatformUpgrade() + ? " <-- " + currentPlatform(node.node()) : "") + (node.needsOsUpgrade() && node.isAllowedDown() ? ", upgrading OS (" + node.node().wantedOsVersion() + " <-- " + node.node().currentOsVersion() + ")" @@ -548,6 +548,17 @@ public class InternalStepRunner implements StepRunner { : " has config generation " + service.currentGeneration() + ", wanted is " + node.wantedConfigGeneration()))); } + + private String wantedPlatform(Node node) { + return node.wantedDockerImage().repository() + ":" + node.wantedVersion(); + } + + private String currentPlatform(Node node) { + String currentRepo = node.currentDockerImage().repository(); + String wantedRepo = node.wantedDockerImage().repository(); + return (currentRepo.equals(wantedRepo) ? "" : currentRepo + ":") + node.currentVersion(); + } + private String humanize(Node.ServiceState state) { switch (state) { case allowedDown: return "allowed to be DOWN"; @@ -941,8 +952,8 @@ public class InternalStepRunner implements StepRunner { Duration endpoint() { return Duration.ofMinutes(15); } Duration endpointCertificate() { return Duration.ofMinutes(20); } Duration tester() { return Duration.ofMinutes(30); } - Duration nodesDown() { return Duration.ofMinutes(system.isCd() ? 20 : 60); } - Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 20 : 120); } + Duration nodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 60); } + Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 120); } Duration testerCertificate() { return Duration.ofMinutes(300); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index 4e461534cc0..855cbdb5337 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; @@ -74,11 +74,11 @@ public class NameServiceForwarder { NameServiceQueue queue = db.readNameServiceQueue(); var queued = queue.requests().size(); if (queued >= maxQueuedRequests) { - log.log(LogLevel.WARNING, "Queue is at capacity (size: " + queued + "), dropping older " + + log.log(Level.WARNING, "Queue is at capacity (size: " + queued + "), dropping older " + "requests. This likely means that the name service is not successfully " + "executing requests"); } - log.log(LogLevel.INFO, "Queueing name service request: " + request); + log.log(Level.INFO, "Queueing name service request: " + request); db.writeNameServiceQueue(queue.with(request, priority).last(maxQueuedRequests)); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java index 684fb091d92..ee5d50414c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.dns; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import java.util.ArrayList; @@ -78,7 +78,7 @@ public class NameServiceQueue { request.dispatchTo(nameService); queue.requests.poll(); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to execute " + request + ": " + e.getMessage() + + log.log(Level.WARNING, "Failed to execute " + request + ": " + e.getMessage() + ", request will be retried"); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index d1f2087ce12..12b52d1bf2f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.TenantController; @@ -55,7 +55,7 @@ public class ContactInformationMaintainer extends Maintainer { throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'."); } } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to update contact information for " + tenant + ": " + + log.log(Level.WARNING, "Failed to update contact information for " + tenant + ": " + Exceptions.toMessageString(e) + ". Retrying in " + maintenanceInterval()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 1ce1e7f4c0c..a4078f7e028 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -80,7 +80,7 @@ public class DeploymentMetricsMaintainer extends Maintainer { try { pool.awaitTermination(30, TimeUnit.MINUTES); if (lastException.get() != null) { - log.log(LogLevel.WARNING, + log.log(Level.WARNING, String.format("Failed to gather metrics for %d/%d applications. Retrying in %s. Last error: %s", failures.get(), attempts.get(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 23e3149ec1e..e8391b8aded 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner; @@ -86,7 +86,7 @@ public class JobRunner extends Maintainer { .ifPresent(run -> controller().applications().deploymentTrigger().notifyOfCompletion(id.application())); } catch (Exception e) { - log.log(LogLevel.WARNING, "Exception finishing " + id, e); + log.log(Level.WARNING, "Exception finishing " + id, e); } } @@ -118,7 +118,7 @@ public class JobRunner extends Maintainer { // Something else is already advancing this step, or a prerequisite -- try again later! } catch (RuntimeException e) { - log.log(LogLevel.WARNING, "Exception attempting to advance " + step + " of " + id, e); + log.log(Level.WARNING, "Exception attempting to advance " + step + " of " + id, e); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 9c414ce8348..dfd35d52b2d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import com.yahoo.vespa.hosted.controller.deployment.JobList; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; @@ -20,6 +21,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -38,19 +40,11 @@ public class MetricsReporter extends Maintainer { public static final String DEPLOYMENT_FAILING_UPGRADES = "deployment.failingUpgrades"; public static final String DEPLOYMENT_BUILD_AGE_SECONDS = "deployment.buildAgeSeconds"; public static final String DEPLOYMENT_WARNINGS = "deployment.warnings"; - public static final String NODES_FAILING_SYSTEM_UPGRADE = "deployment.nodesFailingSystemUpgrade"; - public static final String NODES_FAILING_OS_UPGRADE = "deployment.nodesFailingOsUpgrade"; + public static final String OS_CHANGE_DURATION = "deployment.osChangeDuration"; + public static final String PLATFORM_CHANGE_DURATION = "deployment.platformChangeDuration"; public static final String REMAINING_ROTATIONS = "remaining_rotations"; public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; - // The time a node belonging to a system application can spend from being told to upgrade until the upgrade is - // completed. Nodes exceeding this time are counted as failures. - private static final Duration NODE_UPGRADE_TIMEOUT = Duration.ofMinutes(90); - - // The time a single node can spend performing an OS upgrade after being told to upgrade. Nodes exceeding this time - // multiplied by the number of nodes upgrading are counted as failures. - private static final Duration OS_UPGRADE_TIME_ALLOWANCE_PER_NODE = Duration.ofMinutes(30); - private final Metric metric; private final Clock clock; @@ -65,7 +59,7 @@ public class MetricsReporter extends Maintainer { reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); - reportNodesFailingUpgrade(); + reportChangeDurations(); } private void reportRemainingRotations() { @@ -107,43 +101,36 @@ public class MetricsReporter extends Maintainer { metric.createContext(Map.of())); } - private void reportNodesFailingUpgrade() { - metric.set(NODES_FAILING_SYSTEM_UPGRADE, nodesFailingSystemUpgrade(), metric.createContext(Map.of())); - metric.set(NODES_FAILING_OS_UPGRADE, nodesFailingOsUpgrade(), metric.createContext(Map.of())); + private void reportChangeDurations() { + Map<NodeVersion, Duration> platformChangeDurations = platformChangeDurations(); + Map<NodeVersion, Duration> osChangeDurations = osChangeDurations(); + platformChangeDurations.forEach((nodeVersion, duration) -> { + metric.set(PLATFORM_CHANGE_DURATION, duration.toSeconds(), metric.createContext(dimensions(nodeVersion))); + }); + osChangeDurations.forEach((nodeVersion, duration) -> { + metric.set(OS_CHANGE_DURATION, duration.toSeconds(), metric.createContext(dimensions(nodeVersion))); + }); } - private int nodesFailingSystemUpgrade() { - if (!controller().versionStatus().isUpgrading()) return 0; - return nodesFailingUpgrade(controller().versionStatus().versions(), (vespaVersion) -> { - if (vespaVersion.confidence() == VespaVersion.Confidence.broken) return NodeVersions.EMPTY; - return vespaVersion.nodeVersions(); - }, NODE_UPGRADE_TIMEOUT); + private Map<NodeVersion, Duration> platformChangeDurations() { + return changeDurations(controller().versionStatus().versions(), VespaVersion::nodeVersions); } - private int nodesFailingOsUpgrade() { - var allNodeVersions = controller().osVersionStatus().versions().values(); - var totalTimeout = 0L; - for (var nodeVersions : allNodeVersions) { - for (var nodeVersion : nodeVersions.asMap().values()) { - if (!nodeVersion.changing()) continue; - totalTimeout += OS_UPGRADE_TIME_ALLOWANCE_PER_NODE.toMillis(); - } - } - return nodesFailingUpgrade(allNodeVersions, Function.identity(), Duration.ofMillis(totalTimeout)); + private Map<NodeVersion, Duration> osChangeDurations() { + return changeDurations(controller().osVersionStatus().versions().values(), Function.identity()); } - private <V> int nodesFailingUpgrade(Collection<V> collection, Function<V, NodeVersions> nodeVersionsFunction, Duration timeout) { - var nodesFailingUpgrade = 0; - var acceptableInstant = clock.instant().minus(timeout); - for (var object : collection) { - for (var nodeVersion : nodeVersionsFunction.apply(object).asMap().values()) { - if (!nodeVersion.changing()) continue; - if (nodeVersion.changedAt().isBefore(acceptableInstant)) nodesFailingUpgrade++; + private <V> Map<NodeVersion, Duration> changeDurations(Collection<V> versions, Function<V, NodeVersions> versionsGetter) { + var now = clock.instant(); + var durations = new HashMap<NodeVersion, Duration>(); + for (var version : versions) { + for (var nodeVersion : versionsGetter.apply(version).asMap().values()) { + durations.put(nodeVersion, nodeVersion.changeDuration(now)); } } - return nodesFailingUpgrade; + return durations; } - + private static double deploymentFailRatio(DeploymentStatusList statuses) { return statuses.asList().stream() .mapToInt(status -> status.hasFailures() ? 1 : 0) @@ -198,6 +185,11 @@ public class MetricsReporter extends Maintainer { "app",application.application().value() + "." + application.instance().value()); } + private static Map<String, String> dimensions(NodeVersion nodeVersion) { + return Map.of("host", nodeVersion.hostname().value(), + "zone", nodeVersion.zone().value()); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java index 83460d8f05a..e9ff039f8ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; @@ -47,7 +47,7 @@ public class NameServiceDispatcher extends Maintainer { var dispatched = queue.last(requestCount); if (!dispatched.requests().isEmpty()) { - log.log(LogLevel.INFO, "Dispatched name service request(s) in " + + log.log(Level.INFO, "Dispatched name service request(s) in " + Duration.between(instant, clock.instant()) + ": " + dispatched.requests()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java index cf06489b8c1..4b4d2c8698d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.yolean.Exceptions; @@ -23,7 +23,7 @@ public class OsVersionStatusUpdater extends Maintainer { OsVersionStatus newStatus = OsVersionStatus.compute(controller()); controller().updateOsVersionStatus(newStatus); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + + log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + ". Retrying in " + maintenanceInterval()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java index 35131d3db80..68f8e5464a0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RotationStatusUpdater.java @@ -1,7 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -68,7 +68,7 @@ public class RotationStatusUpdater extends Maintainer { try { pool.awaitTermination(30, TimeUnit.SECONDS); if (lastException.get() != null) { - log.log(LogLevel.WARNING, String.format("Failed to get global routing status of %d/%d applications. Retrying in %s. Last error: ", + log.log(Level.WARNING, String.format("Failed to get global routing status of %d/%d applications. Retrying in %s. Last error: ", failures.get(), attempts.get(), maintenanceInterval()), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index 565acf6ebd5..22030931944 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.yolean.Exceptions; @@ -27,7 +27,7 @@ public class VersionStatusUpdater extends Maintainer { VersionStatus newStatus = VersionStatus.compute(controller()); controller().updateVersionStatus(newStatus); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + + log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + ". Retrying in " + maintenanceInterval()); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 77cdd8d342d..db0866de5b6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -9,7 +9,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; @@ -357,7 +357,7 @@ public class CuratorDb { readApplication(id).ifPresent(applications::add); } catch (Exception e) { if (canFail) { - log.log(LogLevel.ERROR, "Failed to read application '" + id + "', this must be fixed through " + + log.log(Level.SEVERE, "Failed to read application '" + id + "', this must be fixed through " + "manual intervention", e); } else { throw e; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java index 6e9def40e44..f9f8de96591 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java @@ -11,8 +11,6 @@ import com.yahoo.slime.Inspector; import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.NodeVersions; -import java.time.Instant; - /** * Serializer for {@link com.yahoo.vespa.hosted.controller.versions.NodeVersion}. * @@ -30,7 +28,7 @@ public class NodeVersionSerializer { private static final String hostnameField = "hostname"; private static final String zoneField = "zone"; private static final String wantedVersionField = "wantedVersion"; - private static final String changedAtField = "changedAt"; + private static final String suspendedAtField = "suspendedAt"; public void nodeVersionsToSlime(NodeVersions nodeVersions, Cursor array) { for (var nodeVersion : nodeVersions.asMap().values()) { @@ -38,7 +36,8 @@ public class NodeVersionSerializer { nodeVersionObject.setString(hostnameField, nodeVersion.hostname().value()); nodeVersionObject.setString(zoneField, nodeVersion.zone().value()); nodeVersionObject.setString(wantedVersionField, nodeVersion.wantedVersion().toFullString()); - nodeVersionObject.setLong(changedAtField, nodeVersion.changedAt().toEpochMilli()); + nodeVersion.suspendedAt().ifPresent(suspendedAt -> nodeVersionObject.setLong(suspendedAtField, + suspendedAt.toEpochMilli())); } } @@ -48,8 +47,8 @@ public class NodeVersionSerializer { var hostname = HostName.from(entry.field(hostnameField).asString()); var zone = ZoneId.from(entry.field(zoneField).asString()); var wantedVersion = Version.fromString(entry.field(wantedVersionField).asString()); - var changedAt = Instant.ofEpochMilli(entry.field(changedAtField).asLong()); - nodeVersions.put(hostname, new NodeVersion(hostname, zone, version, wantedVersion, changedAt)); + var suspendedAt = Serializers.optionalInstant(entry.field(suspendedAtField)); + nodeVersions.put(hostname, new NodeVersion(hostname, zone, version, wantedVersion, suspendedAt)); }); return new NodeVersions(nodeVersions.build()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 95867b3786c..d10c4dd226b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.proxy; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.jdisc.http.HttpRequest.Method; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; @@ -102,7 +102,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C errorBuilder.append("Talking to server ").append(uri.getHost()); errorBuilder.append(", got ").append(status).append(" ") .append(content).append("\n"); - log.log(LogLevel.DEBUG, () -> String.format("Got response from %s with status code %d and content:\n %s", + log.log(Level.FINE, () -> String.format("Got response from %s with status code %d and content:\n %s", uri.getHost(), status, content)); return Optional.empty(); } @@ -118,7 +118,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C } catch (Exception e) { errorBuilder.append("Talking to server ").append(uri.getHost()); errorBuilder.append(" got exception ").append(e.getMessage()); - log.log(LogLevel.DEBUG, e, () -> "Got exception while sending request to " + uri.getHost()); + log.log(Level.FINE, e, () -> "Got exception while sending request to " + uri.getHost()); return Optional.empty(); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java index 4d3cfacab14..48118087a54 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/AthenzRoleFilter.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.restapi.Path; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; @@ -70,7 +70,7 @@ public class AthenzRoleFilter extends JsonSecurityRequestFilterBase { } } catch (Exception e) { - logger.log(LogLevel.INFO, () -> "Exception mapping Athenz principal to roles: " + Exceptions.toMessageString(e)); + logger.log(Level.INFO, () -> "Exception mapping Athenz principal to roles: " + Exceptions.toMessageString(e)); } return Optional.empty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index ec95f0bbb4c..2b9fe708df6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -6,7 +6,7 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.role.Action; import com.yahoo.vespa.hosted.controller.api.role.Enforcer; @@ -52,7 +52,7 @@ public class ControllerAuthorizationFilter extends JsonSecurityRequestFilterBase return Optional.empty(); } catch (Exception e) { - log.log(LogLevel.WARNING, "Exception evaluating access control: ", e); + log.log(Level.WARNING, "Exception evaluating access control: ", e); } return Optional.of(new ErrorResponse(Response.Status.FORBIDDEN, "Access denied")); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java index 4916b7b24a1..1f515035fe7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilter.java @@ -9,7 +9,7 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.TenantName; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.KeyUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -59,7 +59,7 @@ public class SignatureFilter extends JsonSecurityRequestFilterBase { }); } catch (Exception e) { - logger.log(LogLevel.DEBUG, () -> "Exception verifying signed request: " + Exceptions.toMessageString(e)); + logger.log(Level.FINE, () -> "Exception verifying signed request: " + Exceptions.toMessageString(e)); } return Optional.empty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java index 9bd3b70cfab..5ea277f5101 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.restapi.systemflags; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.SystemName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.Flags; @@ -65,7 +65,7 @@ class SystemFlagsDeployer { try { results.add(future.get()); } catch (InterruptedException | ExecutionException e) { - log.log(LogLevel.ERROR, String.format("Failed to deploy flags for target '%s': %s", target, e.getMessage()), e); + log.log(Level.SEVERE, String.format("Failed to deploy flags for target '%s': %s", target, e.getMessage()), e); throw new RuntimeException(e); } }); @@ -85,7 +85,7 @@ class SystemFlagsDeployer { currentFlagData = lookupTable(client.listFlagData(target)); definedFlags = client.listDefinedFlags(target); } catch (Exception e) { - log.log(LogLevel.WARNING, String.format("Failed to list flag data for target '%s': %s", target, e.getMessage()), e); + log.log(Level.WARNING, String.format("Failed to list flag data for target '%s': %s", target, e.getMessage()), e); return new SystemFlagsDeployResult(List.of(OperationError.listFailed(e.getMessage(), target))); } @@ -119,7 +119,7 @@ class SystemFlagsDeployer { dryRunFlagDataValidation(data); } } catch (Exception e) { - log.log(LogLevel.WARNING, String.format("Failed to put flag '%s' for target '%s': %s", data.id(), target, e.getMessage()), e); + log.log(Level.WARNING, String.format("Failed to put flag '%s' for target '%s': %s", data.id(), target, e.getMessage()), e); errors.add(OperationError.createFailed(e.getMessage(), target, data)); return; } @@ -145,7 +145,7 @@ class SystemFlagsDeployer { dryRunFlagDataValidation(wantedData); } } catch (Exception e) { - log.log(LogLevel.WARNING, String.format("Failed to update flag '%s' for target '%s': %s", wantedData.id(), target, e.getMessage()), e); + log.log(Level.WARNING, String.format("Failed to update flag '%s' for target '%s': %s", wantedData.id(), target, e.getMessage()), e); errors.add(OperationError.updateFailed(e.getMessage(), target, wantedData)); return; } @@ -167,7 +167,7 @@ class SystemFlagsDeployer { try { client.deleteFlagData(target, id); } catch (Exception e) { - log.log(LogLevel.WARNING, String.format("Failed to delete flag '%s' for target '%s': %s", id, target, e.getMessage()), e); + log.log(Level.WARNING, String.format("Failed to delete flag '%s' for target '%s': %s", id, target, e.getMessage()), e); errors.add(OperationError.deleteFailed(e.getMessage(), target, id)); return; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java index 2f7716aaff0..43a03bf10f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java @@ -6,7 +6,7 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.logging.AccessLog; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.JacksonJsonResponse; import com.yahoo.restapi.Path; @@ -66,7 +66,7 @@ public class SystemFlagsHandler extends LoggingRequestHandler { return new JacksonJsonResponse<>(200, result.toWire()); } catch (Exception e) { String errorMessage = "System flags deploy failed: " + e.getMessage(); - log.log(LogLevel.ERROR, errorMessage, e); + log.log(Level.SEVERE, errorMessage, e); return ErrorResponse.internalServerError(errorMessage); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java index d82cddb4779..4a00add411f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java @@ -5,8 +5,10 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; +import java.time.Duration; import java.time.Instant; import java.util.Objects; +import java.util.Optional; /** * Version information for a node allocated to a {@link com.yahoo.vespa.hosted.controller.application.SystemApplication}. @@ -21,14 +23,15 @@ public class NodeVersion { private final ZoneId zone; private final Version currentVersion; private final Version wantedVersion; - private final Instant changedAt; + private final Optional<Instant> suspendedAt; - public NodeVersion(HostName hostname, ZoneId zone, Version currentVersion, Version wantedVersion, Instant changedAt) { + public NodeVersion(HostName hostname, ZoneId zone, Version currentVersion, Version wantedVersion, + Optional<Instant> suspendedAt) { this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); this.zone = Objects.requireNonNull(zone, "zone must be non-null"); this.currentVersion = Objects.requireNonNull(currentVersion, "version must be non-null"); this.wantedVersion = Objects.requireNonNull(wantedVersion, "wantedVersion must be non-null"); - this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null"); + this.suspendedAt = Objects.requireNonNull(suspendedAt, "suspendedAt must be non-null"); } /** Hostname of this */ @@ -51,31 +54,39 @@ public class NodeVersion { return wantedVersion; } - /** Returns whether this is changing (upgrading or downgrading) */ - public boolean changing() { - return !currentVersion.equals(wantedVersion); + /** Returns the duration of the change in this, measured relative to instant */ + public Duration changeDuration(Instant instant) { + if (!changing()) return Duration.ZERO; + if (suspendedAt.isEmpty()) return Duration.ZERO; // Node hasn't suspended to apply the change yet + return Duration.between(suspendedAt.get(), instant).abs(); } - /** The most recent time the version of this changed */ - public Instant changedAt() { - return changedAt; + /** The most recent time the node referenced by this suspended. This is empty if the node is not suspended. */ + public Optional<Instant> suspendedAt() { + return suspendedAt; } /** Returns a copy of this with current version set to given version */ - public NodeVersion withCurrentVersion(Version version, Instant changedAt) { + public NodeVersion withCurrentVersion(Version version) { if (currentVersion.equals(version)) return this; - return new NodeVersion(hostname, zone, version, wantedVersion, changedAt); + return new NodeVersion(hostname, zone, version, wantedVersion, suspendedAt); } /** Returns a copy of this with wanted version set to given version */ public NodeVersion withWantedVersion(Version version) { if (wantedVersion.equals(version)) return this; - return new NodeVersion(hostname, zone, currentVersion, version, changedAt); + return new NodeVersion(hostname, zone, currentVersion, version, suspendedAt); + } + + /** Returns a copy of this with wanted version set to given version */ + public NodeVersion withSuspendedAt(Optional<Instant> suspendedAt) { + if (suspendedAt.equals(this.suspendedAt)) return this; + return new NodeVersion(hostname, zone, currentVersion, wantedVersion, suspendedAt); } @Override public String toString() { - return hostname + ": " + currentVersion + " -> " + wantedVersion + " [zone=" + zone + ", changedAt=" + changedAt + "]"; + return hostname + ": " + currentVersion + " -> " + wantedVersion + " [zone=" + zone + ", suspendedAt=" + suspendedAt.map(Instant::toString).orElse("<not suspended>") + "]"; } @Override @@ -87,12 +98,17 @@ public class NodeVersion { zone.equals(that.zone) && currentVersion.equals(that.currentVersion) && wantedVersion.equals(that.wantedVersion) && - changedAt.equals(that.changedAt); + suspendedAt.equals(that.suspendedAt); } @Override public int hashCode() { - return Objects.hash(hostname, zone, currentVersion, wantedVersion, changedAt); + return Objects.hash(hostname, zone, currentVersion, wantedVersion, suspendedAt); + } + + /** Returns whether this is changing (upgrading or downgrading) */ + private boolean changing() { + return !currentVersion.equals(wantedVersion); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java index 3ab96e03bcd..4ce0a35e96f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java @@ -7,6 +7,7 @@ import com.google.common.collect.ListMultimap; import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -20,12 +21,10 @@ import java.util.function.Predicate; */ public class NodeVersions { - public static final NodeVersions EMPTY = new NodeVersions(ImmutableMap.of()); - private final ImmutableMap<HostName, NodeVersion> nodeVersions; - public NodeVersions(ImmutableMap<HostName, NodeVersion> nodeVersions) { - this.nodeVersions = Objects.requireNonNull(nodeVersions); + public NodeVersions(Map<HostName, NodeVersion> nodeVersions) { + this.nodeVersions = ImmutableMap.copyOf(Objects.requireNonNull(nodeVersions)); } public Map<HostName, NodeVersion> asMap() { @@ -48,7 +47,7 @@ public class NodeVersions { /** Returns a copy of this containing only node versions of given version */ public NodeVersions matching(Version version) { - return filter(nodeVersion -> nodeVersion.currentVersion().equals(version)); + return copyOf(nodeVersions.values(), nodeVersion -> nodeVersion.currentVersion().equals(version)); } /** Returns number of node versions in this */ @@ -56,31 +55,6 @@ public class NodeVersions { return nodeVersions.size(); } - /** Returns a copy of this containing only the given node versions */ - public NodeVersions with(List<NodeVersion> nodeVersions) { - var newNodeVersions = ImmutableMap.<HostName, NodeVersion>builder(); - for (var nodeVersion : nodeVersions) { - var existing = this.nodeVersions.get(nodeVersion.hostname()); - if (existing != null) { - newNodeVersions.put(nodeVersion.hostname(), existing.withCurrentVersion(nodeVersion.currentVersion(), - nodeVersion.changedAt()) - .withWantedVersion(nodeVersion.wantedVersion())); - } else { - newNodeVersions.put(nodeVersion.hostname(), nodeVersion); - } - } - return new NodeVersions(newNodeVersions.build()); - } - - private NodeVersions filter(Predicate<NodeVersion> predicate) { - var newNodeVersions = ImmutableMap.<HostName, NodeVersion>builder(); - for (var kv : nodeVersions.entrySet()) { - if (!predicate.test(kv.getValue())) continue; - newNodeVersions.put(kv.getKey(), kv.getValue()); - } - return new NodeVersions(newNodeVersions.build()); - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -94,4 +68,21 @@ public class NodeVersions { return Objects.hash(nodeVersions); } + public static NodeVersions copyOf(List<NodeVersion> nodeVersions) { + return copyOf(nodeVersions, (ignored) -> true); + } + + public static NodeVersions copyOf(Map<HostName, NodeVersion> nodeVersions) { + return new NodeVersions(nodeVersions); + } + + private static NodeVersions copyOf(Collection<NodeVersion> nodeVersions, Predicate<NodeVersion> predicate) { + var newNodeVersions = ImmutableMap.<HostName, NodeVersion>builder(); + for (var nodeVersion : nodeVersions) { + if (!predicate.test(nodeVersion)) continue; + newNodeVersions.put(nodeVersion.hostname(), nodeVersion); + } + return new NodeVersions(newNodeVersions.build()); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index 1773a9c122e..1dffd1383bd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -16,7 +16,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -59,9 +58,7 @@ public class OsVersionStatus { /** Compute the current OS versions in this system. This is expensive and should be called infrequently */ public static OsVersionStatus compute(Controller controller) { - var osVersionStatus = controller.osVersionStatus(); var osVersions = new HashMap<OsVersion, List<NodeVersion>>(); - var now = controller.clock().instant(); controller.osVersions().forEach(osVersion -> osVersions.put(osVersion, new ArrayList<>())); for (var application : SystemApplication.all()) { @@ -71,19 +68,16 @@ public class OsVersionStatus { .targetVersionsOf(zone.getId()) .osVersion(application.nodeType()) .orElse(Version.emptyVersion); - controller.serviceRegistry().configServer().nodeRepository() - .list(zone.getId(), application.id()).stream() - .filter(node -> OsUpgrader.eligibleForUpgrade(node, application)) - .map(node -> new NodeVersion(node.hostname(), zone.getId(), node.currentOsVersion(), targetOsVersion, now)) - .forEach(nodeVersion -> { - var newNodeVersion = osVersionStatus.of(nodeVersion.hostname()) - .map(nv -> nv.withCurrentVersion(nodeVersion.currentVersion(), now) - .withWantedVersion(nodeVersion.wantedVersion())) - .orElse(nodeVersion); - var version = new OsVersion(newNodeVersion.currentVersion(), zone.getCloudName()); - osVersions.putIfAbsent(version, new ArrayList<>()); - osVersions.get(version).add(newNodeVersion); - }); + + for (var node : controller.serviceRegistry().configServer().nodeRepository().list(zone.getId(), application.id())) { + if (!OsUpgrader.eligibleForUpgrade(node, application)) continue; + var suspendedAt = node.suspendedSince(); + var nodeVersion = new NodeVersion(node.hostname(), zone.getId(), node.currentOsVersion(), + targetOsVersion, suspendedAt); + var osVersion = new OsVersion(nodeVersion.currentVersion(), zone.getCloudName()); + osVersions.putIfAbsent(osVersion, new ArrayList<>()); + osVersions.get(osVersion).add(nodeVersion); + } } } @@ -98,15 +92,6 @@ public class OsVersionStatus { return new OsVersionStatus(newOsVersions.build()); } - /** Returns version of node identified by given host name */ - private Optional<NodeVersion> of(HostName hostname) { - return versions.values().stream() - .map(nodeVersions -> nodeVersions.asMap().get(hostname)) - .map(Optional::ofNullable) - .flatMap(Optional::stream) - .findFirst(); - } - private static List<ZoneApi> zonesToUpgrade(Controller controller) { return controller.zoneRegistry().osUpgradePolicies().stream() .flatMap(upgradePolicy -> upgradePolicy.asList().stream()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 0868d7ca695..19b98ae12cc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -6,25 +6,17 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ListMultimap; import com.yahoo.component.Version; import com.yahoo.config.provision.HostName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.application.ApplicationList; -import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; -import com.yahoo.vespa.hosted.controller.deployment.JobStatus; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.maintenance.SystemUpgrader; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.logging.Level; @@ -149,10 +141,7 @@ public class VersionStatus { } private static NodeVersions findSystemApplicationVersions(Controller controller) { - var nodeVersions = controller.versionStatus().systemVersion() - .map(VespaVersion::nodeVersions) - .orElse(NodeVersions.EMPTY); - var newNodeVersions = new ArrayList<NodeVersion>(); + var nodeVersions = new LinkedHashMap<HostName, NodeVersion>(); for (var zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) { for (var application : SystemApplication.all()) { var nodes = controller.serviceRegistry().configServer().nodeRepository() @@ -162,18 +151,19 @@ public class VersionStatus { if (nodes.isEmpty()) continue; var configConverged = application.configConvergedIn(zone.getId(), controller, Optional.empty()); if (!configConverged) { - log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone.getId() + + log.log(Level.WARNING, "Config for " + application.id() + " in " + zone.getId() + " has not converged"); } - var now = controller.clock().instant(); for (var node : nodes) { // Only use current node version if config has converged - Version version = configConverged ? node.currentVersion() : controller.systemVersion(); - newNodeVersions.add(new NodeVersion(node.hostname(), zone.getId(), version, node.wantedVersion(), now)); + var version = configConverged ? node.currentVersion() : controller.systemVersion(); + var nodeVersion = new NodeVersion(node.hostname(), zone.getId(), version, node.wantedVersion(), + node.suspendedSince()); + nodeVersions.put(nodeVersion.hostname(), nodeVersion); } } } - return nodeVersions.with(newNodeVersions); + return NodeVersions.copyOf(nodeVersions); } private static ListMultimap<ControllerVersion, HostName> findControllerVersions(Controller controller) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 775eb2a4d75..189bd5156e3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -223,7 +223,7 @@ public final class ControllerTester { if (!application.hasApplicationPackage()) { configServer().nodeRepository().upgrade(zone.getId(), application.nodeType(), version); } - configServer().setVersion(application.id(), zone.getId(), version); + configServer().setVersion(version, application.id(), zone.getId()); configServer().convergeServices(application.id(), zone.getId()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 589229b32d4..0dccea41ae1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; @@ -85,9 +85,9 @@ public class DeploymentTester { outstandingChangeDeployer = new OutstandingChangeDeployer(tester.controller(), maintenanceInterval, jobControl); // Get deployment job logs to stderr. - Logger.getLogger("").setLevel(LogLevel.DEBUG); - Logger.getLogger(InternalStepRunner.class.getName()).setLevel(LogLevel.DEBUG); - tester.configureDefaultLogHandler(handler -> handler.setLevel(LogLevel.DEBUG)); + Logger.getLogger("").setLevel(Level.FINE); + Logger.getLogger(InternalStepRunner.class.getName()).setLevel(Level.FINE); + tester.configureDefaultLogHandler(handler -> handler.setLevel(Level.FINE)); // Mock Athenz domain to allow launch of service AthenzDbMock.Domain domain = tester.athenzDb().getOrCreateDomain(new com.yahoo.vespa.athenz.api.AthenzDomain(ATHENZ_DOMAIN)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 99325ff909d..07c643070a0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -201,9 +201,9 @@ public class InternalStepRunnerTest { // Node is down too long in system test, and no nodes go down in staging. tester.runner().run(); - tester.configServer().setVersion(app.testerId().id(), JobType.systemTest.zone(system()), tester.controller().systemVersion()); + tester.configServer().setVersion(tester.controller().systemVersion(), app.testerId().id(), JobType.systemTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); - tester.configServer().setVersion(app.testerId().id(), JobType.stagingTest.zone(system()), tester.controller().systemVersion()); + tester.configServer().setVersion(tester.controller().systemVersion(), app.testerId().id(), JobType.stagingTest.zone(system())); tester.configServer().convergeServices(app.testerId().id(), JobType.stagingTest.zone(system())); tester.runner().run(); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); @@ -212,8 +212,8 @@ public class InternalStepRunnerTest { Node systemTestNode = tester.configServer().nodeRepository().list(JobType.systemTest.zone(system()), app.instanceId()).iterator().next(); tester.clock().advance(InternalStepRunner.Timeouts.of(system()).noNodesDown().minus(Duration.ofSeconds(1))); - tester.configServer().nodeRepository().putByHostname(JobType.systemTest.zone(system()), - new Node.Builder(systemTestNode) + tester.configServer().nodeRepository().putNodes(JobType.systemTest.zone(system()), + new Node.Builder(systemTestNode) .serviceState(Node.ServiceState.allowedDown) .suspendedSince(tester.clock().instant()) .build()); @@ -377,7 +377,7 @@ public class InternalStepRunnerTest { assertEquals(applicationPackage.hash(), tester.configServer().application(app.instanceId(), zone).get().applicationPackage().hash()); assertEquals(otherPackage.hash(), tester.configServer().application(app.instanceId(), JobType.perfUsEast3.zone(system())).get().applicationPackage().hash()); - tester.configServer().setVersion(app.instanceId(), zone, version); + tester.configServer().setVersion(version, app.instanceId(), zone); tester.runner().run(); assertEquals(1, tester.jobs().active().size()); assertEquals(version, tester.instance(app.instanceId()).deployments().get(zone).version()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 9e9d27fd744..0fb2451ccba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -100,23 +100,23 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer public void provision(ZoneId zone, ApplicationId application) { Node parent = nodeRepository().list(zone, SystemApplication.tenantHost.id()).stream().findAny() .orElseThrow(() -> new IllegalStateException("No parent hosts in " + zone)); - nodeRepository().putByHostname(zone, new Node.Builder().hostname(hostFor(application, zone)) - .state(Node.State.reserved) - .type(NodeType.tenant) - .owner(application) - .parentHostname(parent.hostname()) - .currentVersion(initialVersion) - .wantedVersion(initialVersion) - .currentDockerImage(initialDockerImage) - .wantedDockerImage(initialDockerImage) - .currentOsVersion(Version.emptyVersion) - .wantedOsVersion(Version.emptyVersion) - .resources(new NodeResources(2, 8, 50, 1, slow, remote)) - .serviceState(Node.ServiceState.unorchestrated) - .flavor("d-2-8-50") - .clusterId("cluster") - .clusterType(Node.ClusterType.container) - .build()); + nodeRepository().putNodes(zone, new Node.Builder().hostname(hostFor(application, zone)) + .state(Node.State.reserved) + .type(NodeType.tenant) + .owner(application) + .parentHostname(parent.hostname()) + .currentVersion(initialVersion) + .wantedVersion(initialVersion) + .currentDockerImage(initialDockerImage) + .wantedDockerImage(initialDockerImage) + .currentOsVersion(Version.emptyVersion) + .wantedOsVersion(Version.emptyVersion) + .resources(new NodeResources(2, 8, 50, 1, slow, remote)) + .serviceState(Node.ServiceState.unorchestrated) + .flavor("d-2-8-50") + .clusterId("cluster") + .clusterType(Node.ClusterType.container) + .build()); } public HostName hostFor(ApplicationId application, ZoneId zone) { @@ -146,7 +146,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer .currentOsVersion(Version.emptyVersion).wantedOsVersion(Version.emptyVersion) .build()) .collect(Collectors.toList()); - nodeRepository().putByHostname(zone, nodes); + nodeRepository().putNodes(zone, nodes); convergeServices(application.id(), zone); } } @@ -178,36 +178,34 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } /** Set version for an application in a given zone */ - public void setVersion(ApplicationId application, ZoneId zone, Version version) { - setVersion(application, zone, version, -1, false); + public void setVersion(Version version, ApplicationId application, ZoneId zone) { + setVersion(zone, nodeRepository.list(zone, application), version, false); } /** Set version for nodeCount number of nodes in application in a given zone */ - public void setVersion(ApplicationId application, ZoneId zone, Version version, int nodeCount) { - setVersion(application, zone, version, nodeCount, false); + public void setVersion(Version version, List<Node> nodes, ZoneId zone) { + setVersion(zone, nodes, version, false); } /** Set OS version for an application in a given zone */ - public void setOsVersion(ApplicationId application, ZoneId zone, Version version) { - setOsVersion(application, zone, version, -1); + public void setOsVersion(Version version, ApplicationId application, ZoneId zone) { + setVersion(zone, nodeRepository.list(zone, application), version, true); } /** Set OS version for an application in a given zone */ - public void setOsVersion(ApplicationId application, ZoneId zone, Version version, int nodeCount) { - setVersion(application, zone, version, nodeCount, true); + public void setOsVersion(Version version, List<Node> nodes, ZoneId zone) { + setVersion(zone, nodes, version, true); } - private void setVersion(ApplicationId application, ZoneId zone, Version version, int nodeCount, boolean osVersion) { - int n = 0; - for (Node node : nodeRepository().list(zone, application)) { + private void setVersion(ZoneId zone, List<Node> nodes, Version version, boolean osVersion) { + for (var node : nodes) { Node newNode; if (osVersion) { newNode = new Node.Builder(node).currentOsVersion(version).wantedOsVersion(version).build(); } else { newNode = new Node.Builder(node).currentVersion(version).wantedVersion(version).build(); } - nodeRepository().putByHostname(zone, newNode); - if (++n == nodeCount) break; + nodeRepository().putNodes(zone, newNode); } } @@ -355,7 +353,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer application.activate(); List<Node> nodes = nodeRepository.list(id.zoneId(), id.applicationId()); for (Node node : nodes) { - nodeRepository.putByHostname(id.zoneId(), new Node.Builder(node) + nodeRepository.putNodes(id.zoneId(), new Node.Builder(node) .state(Node.State.active) .wantedVersion(application.version().get()) .build()); @@ -397,8 +395,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public void deactivate(DeploymentId deployment) throws NotFoundException { ApplicationId applicationId = deployment.applicationId(); - nodeRepository().removeByHostname(deployment.zoneId(), - nodeRepository().list(deployment.zoneId(), applicationId)); + nodeRepository().removeNodes(deployment.zoneId(), + nodeRepository().list(deployment.zoneId(), applicationId)); if ( ! applications.containsKey(deployment)) throw new NotFoundException("No application with id " + applicationId + " exists, cannot deactivate"); applications.remove(deployment); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 4aab21a44fe..1e535e7e9de 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -36,24 +36,33 @@ public class NodeRepositoryMock implements NodeRepository { private final Map<ZoneId, Map<HostName, Node>> nodeRepository = new HashMap<>(); private final Map<ZoneId, TargetVersions> targetVersions = new HashMap<>(); - public void putByHostname(ZoneId zone, List<Node> nodes) { + /** Add or update given nodes in zone */ + public void putNodes(ZoneId zone, List<Node> nodes) { nodeRepository.putIfAbsent(zone, new HashMap<>()); nodeRepository.get(zone).putAll(nodes.stream().collect(Collectors.toMap(Node::hostname, Function.identity()))); } - public void putByHostname(ZoneId zone, Node node) { - putByHostname(zone, Collections.singletonList(node)); + /** Add or update given node in zone */ + public void putNodes(ZoneId zone, Node node) { + putNodes(zone, Collections.singletonList(node)); } - public void removeByHostname(ZoneId zone, List<Node> nodes) { + /** Remove given nodes from zone */ + public void removeNodes(ZoneId zone, List<Node> nodes) { nodes.forEach(node -> nodeRepository.get(zone).remove(node.hostname())); } + /** Remove all nodes in all zones */ public void clear() { nodeRepository.clear(); } + /** Replace nodes in zone with given nodes */ + public void setNodes(ZoneId zone, List<Node> nodes) { + nodeRepository.put(zone, nodes.stream().collect(Collectors.toMap(Node::hostname, Function.identity()))); + } + public Node require(HostName hostName) { return nodeRepository.values().stream() .map(zoneNodes -> zoneNodes.get(hostName)) @@ -62,11 +71,8 @@ public class NodeRepositoryMock implements NodeRepository { .orElseThrow(() -> new NoSuchElementException("No node with the hostname " + hostName + " is known.")); } - public void addNodes(ZoneId zone, List<Node> nodes) { - nodeRepository.put(zone, nodes.stream().collect(Collectors.toMap(Node::hostname, Function.identity()))); - } - - public void addFixedNodes(ZoneId zone) { + /** Replace nodes in zone with a fixed set of nodes */ + public void setFixedNodes(ZoneId zone) { var nodeA = new Node.Builder() .hostname(HostName.from("hostA")) .parentHostname(HostName.from("parentHostA")) @@ -98,7 +104,7 @@ public class NodeRepositoryMock implements NodeRepository { .clusterId("clusterB") .clusterType(Node.ClusterType.container) .build(); - addNodes(zone, List.of(nodeA, nodeB)); + setNodes(zone, List.of(nodeA, nodeB)); } @Override @@ -168,7 +174,7 @@ public class NodeRepositoryMock implements NodeRepository { .stream() .filter(node -> node.type() == type) .map(node -> new Node.Builder(node).wantedVersion(version).build()) - .forEach(node -> putByHostname(zone, node)); + .forEach(node -> putNodes(zone, node)); } @Override @@ -184,7 +190,7 @@ public class NodeRepositoryMock implements NodeRepository { .stream() .filter(node -> node.type() == type) .map(node -> new Node.Builder(node).wantedOsVersion(version).build()) - .forEach(node -> putByHostname(zone, node)); + .forEach(node -> putNodes(zone, node)); } @Override @@ -219,8 +225,8 @@ public class NodeRepositoryMock implements NodeRepository { List<Node> nodes = hostname.map(this::require) .map(Collections::singletonList) .orElse(list(deployment.zoneId(), deployment.applicationId())); - putByHostname(deployment.zoneId(), - nodes.stream().map(modification).collect(Collectors.toList())); + putNodes(deployment.zoneId(), + nodes.stream().map(modification).collect(Collectors.toList())); } public void requestRestart(DeploymentId deployment, Optional<HostName> hostname) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java index 1c66a01db8e..369ecaf5eb7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudEventReporterTest.java @@ -97,7 +97,7 @@ public class CloudEventReporterTest { awsZone1, awsZone2); - tester.configServer().nodeRepository().putByHostname( + tester.configServer().nodeRepository().putNodes( nonAwsZone.getId(), createNodesWithHostnames( "host1.com", @@ -105,7 +105,7 @@ public class CloudEventReporterTest { "host3.com" ) ); - tester.configServer().nodeRepository().putByHostname( + tester.configServer().nodeRepository().putNodes( awsZone1.getId(), createNodesWithHostnames( "host1.com", @@ -113,14 +113,14 @@ public class CloudEventReporterTest { "host3.com" ) ); - tester.configServer().nodeRepository().putByHostname( + tester.configServer().nodeRepository().putNodes( awsZone2.getId(), createNodesWithHostnames( "host4.com", "host5.com" ) ); - tester.configServer().nodeRepository().putByHostname( + tester.configServer().nodeRepository().putNodes( awsZone2.getId(), List.of(createNode("confighost.com", NodeType.confighost)) ); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java index ca624281e94..b19003ff734 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java @@ -54,7 +54,7 @@ public class CostReportMaintainerTest { private void addNodes() { for (var zone : tester.zoneRegistry().zones().all().zones()) { - tester.configServer().nodeRepository().addFixedNodes(zone.getId()); + tester.configServer().nodeRepository().setFixedNodes(zone.getId()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index c00705149e9..df5c3bcbc3c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; @@ -22,12 +23,15 @@ import org.junit.Test; import java.time.Duration; import java.util.Comparator; import java.util.List; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author mortent @@ -226,46 +230,51 @@ public class MetricsReporterTest { public void nodes_failing_system_upgrade() { var tester = new ControllerTester(); var reporter = createReporter(tester.controller()); - var zone1 = ZoneApiMock.fromId("prod.eu-west-1"); - tester.zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(zone1)); + var zone = ZoneId.from("prod.eu-west-1"); + tester.zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(ZoneApiMock.from(zone))); var systemUpgrader = new SystemUpgrader(tester.controller(), Duration.ofDays(1), new JobControl(tester.curator())); - tester.configServer().bootstrap(List.of(zone1.getId()), SystemApplication.configServer); + tester.configServer().bootstrap(List.of(zone), SystemApplication.configServer); // System on initial version var version0 = Version.fromString("7.0"); tester.upgradeSystem(version0); reporter.maintain(); - assertEquals(0, getNodesFailingUpgrade()); + var hosts = tester.configServer().nodeRepository().list(zone, SystemApplication.configServer.id()); + assertPlatformChangeDuration(Duration.ZERO, hosts); for (var version : List.of(Version.fromString("7.1"), Version.fromString("7.2"))) { // System starts upgrading to next version tester.upgradeController(version); reporter.maintain(); - assertEquals(0, getNodesFailingUpgrade()); + assertPlatformChangeDuration(Duration.ZERO, hosts); systemUpgrader.maintain(); // 30 minutes pass and nothing happens tester.clock().advance(Duration.ofMinutes(30)); - tester.computeVersionStatus(); - reporter.maintain(); - assertEquals(0, getNodesFailingUpgrade()); + runAll(tester::computeVersionStatus, reporter); + assertPlatformChangeDuration(Duration.ZERO, hosts); // 1/3 nodes upgrade within timeout assertEquals("Wanted version is raised for all nodes", version, - tester.configServer().nodeRepository().list(zone1.getId(), SystemApplication.configServer.id()).stream() - .map(Node::wantedVersion).min(Comparator.naturalOrder()).get()); - tester.configServer().setVersion(SystemApplication.configServer.id(), zone1.getId(), version, 1); - tester.clock().advance(Duration.ofMinutes(60).plus(Duration.ofSeconds(1))); - tester.computeVersionStatus(); - reporter.maintain(); - assertEquals(2, getNodesFailingUpgrade()); - - // 3/3 nodes upgrade - tester.configServer().setVersion(SystemApplication.configServer.id(), zone1.getId(), version); - tester.computeVersionStatus(); - reporter.maintain(); - assertEquals(0, getNodesFailingUpgrade()); + getNodes(zone, hosts, tester).stream() + .map(Node::wantedVersion) + .min(Comparator.naturalOrder()) + .get()); + suspend(hosts, zone, tester); + var firstHost = hosts.get(0); + upgradeTo(version, List.of(firstHost), zone, tester); + + // 2/3 spend their budget and are reported as failures + tester.clock().advance(Duration.ofHours(1)); + runAll(tester::computeVersionStatus, reporter); + assertPlatformChangeDuration(Duration.ZERO, List.of(firstHost)); + assertPlatformChangeDuration(Duration.ofHours(1), hosts.subList(1, hosts.size())); + + // Remaining nodes eventually upgrade + upgradeTo(version, hosts.subList(1, hosts.size()), zone, tester); + runAll(tester::computeVersionStatus, reporter); + assertPlatformChangeDuration(Duration.ZERO, hosts); assertEquals(version, tester.controller().systemVersion()); } } @@ -274,64 +283,124 @@ public class MetricsReporterTest { public void nodes_failing_os_upgrade() { var tester = new ControllerTester(); var reporter = createReporter(tester.controller()); - var zone = ZoneApiMock.fromId("prod.eu-west-1"); + var zone = ZoneId.from("prod.eu-west-1"); var cloud = CloudName.defaultName(); - tester.zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.create().upgrade(zone)); + tester.zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.create().upgrade(ZoneApiMock.from(zone))); var osUpgrader = new OsUpgrader(tester.controller(), Duration.ofDays(1), new JobControl(tester.curator()), CloudName.defaultName()); var statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator())); - tester.configServer().bootstrap(List.of(zone.getId()), SystemApplication.configServerHost, SystemApplication.tenantHost); + tester.configServer().bootstrap(List.of(zone), SystemApplication.configServerHost, SystemApplication.tenantHost); // All nodes upgrade to initial OS version var version0 = Version.fromString("8.0"); tester.controller().upgradeOsIn(cloud, version0, false); osUpgrader.maintain(); - tester.configServer().setOsVersion(SystemApplication.tenantHost.id(), zone.getId(), version0); - tester.configServer().setOsVersion(SystemApplication.configServerHost.id(), zone.getId(), version0); - statusUpdater.maintain(); - reporter.maintain(); - assertEquals(0, getNodesFailingOsUpgrade()); + tester.configServer().setOsVersion(version0, SystemApplication.tenantHost.id(), zone); + tester.configServer().setOsVersion(version0, SystemApplication.configServerHost.id(), zone); + runAll(statusUpdater, reporter); + List<Node> hosts = tester.configServer().nodeRepository().list(zone); + assertOsChangeDuration(Duration.ZERO, hosts); for (var version : List.of(Version.fromString("8.1"), Version.fromString("8.2"))) { // System starts upgrading to next OS version tester.controller().upgradeOsIn(cloud, version, false); - osUpgrader.maintain(); - statusUpdater.maintain(); - reporter.maintain(); - assertEquals(0, getNodesFailingOsUpgrade()); + runAll(osUpgrader, statusUpdater, reporter); + assertOsChangeDuration(Duration.ZERO, hosts); - // 30 minutes pass and nothing happens - tester.clock().advance(Duration.ofMinutes(30)); - statusUpdater.maintain(); - reporter.maintain(); - assertEquals(0, getNodesFailingOsUpgrade()); + // Over 30 minutes pass and nothing happens + tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1))); + runAll(statusUpdater, reporter); + assertOsChangeDuration(Duration.ZERO, hosts); - // 2/6 nodes upgrade within timeout + // Nodes are told to upgrade, but do not suspend yet assertEquals("Wanted OS version is raised for all nodes", version, - tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.tenantHost.id()).stream() + tester.configServer().nodeRepository().list(zone, SystemApplication.tenantHost.id()).stream() .map(Node::wantedOsVersion).min(Comparator.naturalOrder()).get()); - tester.configServer().setOsVersion(SystemApplication.tenantHost.id(), zone.getId(), version, 2); - tester.clock().advance(Duration.ofMinutes(30 * 3 /* time allowance * node count */).plus(Duration.ofSeconds(1))); - statusUpdater.maintain(); - reporter.maintain(); - assertEquals(4, getNodesFailingOsUpgrade()); - - // 5/6 nodes upgrade - tester.configServer().setOsVersion(SystemApplication.tenantHost.id(), zone.getId(), version); - tester.configServer().setOsVersion(SystemApplication.configServerHost.id(), zone.getId(), version, 2); - statusUpdater.maintain(); - reporter.maintain(); - assertEquals(1, getNodesFailingOsUpgrade()); + assertTrue("No nodes are suspended", tester.controller().serviceRegistry().configServer() + .nodeRepository().list(zone).stream() + .noneMatch(node -> node.serviceState() == Node.ServiceState.allowedDown)); - // Final node upgrades - tester.configServer().setOsVersion(SystemApplication.configServerHost.id(), zone.getId(), version); - statusUpdater.maintain(); - reporter.maintain(); - assertEquals(0, getNodesFailingOsUpgrade()); + // Another 30 minutes pass + tester.clock().advance(Duration.ofMinutes(30)); + runAll(statusUpdater, reporter); + assertOsChangeDuration(Duration.ZERO, hosts); + + // 3/6 hosts suspend + var suspendedHosts = hosts.subList(0, 3); + suspend(suspendedHosts, zone, tester); + runAll(statusUpdater, reporter); + assertOsChangeDuration(Duration.ZERO, hosts); + + // Two hosts spend 20 minutes upgrading + var hostsUpgraded = suspendedHosts.subList(0, 2); + tester.clock().advance(Duration.ofMinutes(20)); + runAll(statusUpdater, reporter); + assertOsChangeDuration(Duration.ofMinutes(20), hostsUpgraded); + upgradeOsTo(version, hostsUpgraded, zone, tester); + runAll(statusUpdater, reporter); + assertOsChangeDuration(Duration.ZERO, hostsUpgraded); + + // One host consumes budget without upgrading + var brokenHost = suspendedHosts.get(2); + tester.clock().advance(Duration.ofMinutes(15)); + runAll(statusUpdater, reporter); + assertOsChangeDuration(Duration.ofMinutes(35), List.of(brokenHost)); + + // Host eventually upgrades and is no longer reported + upgradeOsTo(version, List.of(brokenHost), zone, tester); + runAll(statusUpdater, reporter); + assertOsChangeDuration(Duration.ZERO, List.of(brokenHost)); + + // Remaining hosts suspend and upgrade successfully + var remainingHosts = hosts.subList(3, hosts.size()); + suspend(remainingHosts, zone, tester); + upgradeOsTo(version, remainingHosts, zone, tester); + runAll(statusUpdater, reporter); + assertOsChangeDuration(Duration.ZERO, hosts); } } + private void runAll(Runnable... runnables) { + for (var r : runnables) r.run(); + } + + private void upgradeTo(Version version, List<Node> nodes, ZoneId zone, ControllerTester tester) { + tester.configServer().setVersion(version, nodes, zone); + resume(nodes, zone, tester); + } + + private void upgradeOsTo(Version version, List<Node> nodes, ZoneId zone, ControllerTester tester) { + tester.configServer().setOsVersion(version, nodes, zone); + resume(nodes, zone, tester); + } + + private void resume(List<Node> nodes, ZoneId zone, ControllerTester tester) { + updateNodes(nodes, (builder) -> builder.serviceState(Node.ServiceState.expectedUp).suspendedSince(null), + zone, tester); + } + + private void suspend(List<Node> nodes, ZoneId zone, ControllerTester tester) { + var now = tester.clock().instant(); + updateNodes(nodes, (builder) -> builder.serviceState(Node.ServiceState.allowedDown).suspendedSince(now), + zone, tester); + } + + private List<Node> getNodes(ZoneId zone, List<Node> nodes, ControllerTester tester) { + return tester.configServer().nodeRepository().list(zone, nodes.stream() + .map(Node::hostname) + .collect(Collectors.toList())); + } + + private void updateNodes(List<Node> nodes, UnaryOperator<Node.Builder> builderOps, ZoneId zone, + ControllerTester tester) { + var currentNodes = getNodes(zone, nodes, tester); + var updatedNodes = currentNodes.stream() + .map(node -> builderOps.apply(new Node.Builder(node)).build()) + .collect(Collectors.toList()); + tester.configServer().nodeRepository().putNodes(zone, updatedNodes); + } + private Duration getAverageDeploymentDuration(ApplicationId id) { return Duration.ofSeconds(getMetric(MetricsReporter.DEPLOYMENT_AVERAGE_DURATION, id).longValue()); } @@ -344,12 +413,22 @@ public class MetricsReporterTest { return getMetric(MetricsReporter.DEPLOYMENT_WARNINGS, id).intValue(); } - private int getNodesFailingUpgrade() { - return metrics.getMetric(MetricsReporter.NODES_FAILING_SYSTEM_UPGRADE).intValue(); + private Duration getChangeDuration(String metric, HostName hostname) { + return metrics.getMetric((dimensions) -> hostname.value().equals(dimensions.get("host")), metric) + .map(n -> Duration.ofSeconds(n.longValue())) + .orElseThrow(() -> new IllegalArgumentException("Expected to find metric for " + hostname)); } - private int getNodesFailingOsUpgrade() { - return metrics.getMetric(MetricsReporter.NODES_FAILING_OS_UPGRADE).intValue(); + private void assertPlatformChangeDuration(Duration duration, List<Node> nodes) { + for (var node : nodes) { + assertEquals(duration, getChangeDuration(MetricsReporter.PLATFORM_CHANGE_DURATION, node.hostname())); + } + } + + private void assertOsChangeDuration(Duration duration, List<Node> nodes) { + for (var node : nodes) { + assertEquals(duration, getChangeDuration(MetricsReporter.OS_CHANGE_DURATION, node.hostname())); + } } private Number getMetric(String name, ApplicationId id) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 9d910afd476..baa4b29ac1d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -146,7 +146,7 @@ public class OsUpgraderTest { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); - nodeRepository().putByHostname(zone, new Node.Builder(node).state(Node.State.failed).build()); + nodeRepository().putNodes(zone, new Node.Builder(node).state(Node.State.failed).build()); } /** Simulate OS upgrade of nodes allocated to application. In a real system this is done by the node itself */ @@ -154,7 +154,7 @@ public class OsUpgraderTest { assertWanted(version, application, zones); for (ZoneId zone : zones) { for (Node node : nodesRequiredToUpgrade(zone, application)) { - nodeRepository().putByHostname(zone, new Node.Builder(node).wantedOsVersion(version).currentOsVersion(version).build()); + nodeRepository().putNodes(zone, new Node.Builder(node).wantedOsVersion(version).currentOsVersion(version).build()); } assertCurrent(version, application, zone); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index e0a97d32597..8379722be66 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -64,10 +64,10 @@ public class ResourceMeterMaintainerTest { nonAwsZone, awsZone1, awsZone2); - tester.configServer().nodeRepository().addFixedNodes(nonAwsZone.getId()); - tester.configServer().nodeRepository().addFixedNodes(awsZone1.getId()); - tester.configServer().nodeRepository().addFixedNodes(awsZone2.getId()); - tester.configServer().nodeRepository().putByHostname( + tester.configServer().nodeRepository().setFixedNodes(nonAwsZone.getId()); + tester.configServer().nodeRepository().setFixedNodes(awsZone1.getId()); + tester.configServer().nodeRepository().setFixedNodes(awsZone2.getId()); + tester.configServer().nodeRepository().putNodes( awsZone1.getId(), createNodesInState( Node.State.provisioned, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java index 653a4c5394b..66b125186e0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainerTest.java @@ -46,9 +46,9 @@ public class ResourceTagMaintainerTest { nonAwsZone, awsZone1, awsZone2); - tester.configServer().nodeRepository().addFixedNodes(nonAwsZone.getId()); - tester.configServer().nodeRepository().addFixedNodes(awsZone1.getId()); - tester.configServer().nodeRepository().addFixedNodes(awsZone2.getId()); + tester.configServer().nodeRepository().setFixedNodes(nonAwsZone.getId()); + tester.configServer().nodeRepository().setFixedNodes(awsZone1.getId()); + tester.configServer().nodeRepository().setFixedNodes(awsZone2.getId()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 140c37cdd94..f1031c90344 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -302,7 +302,7 @@ public class SystemUpgraderTest { assertWantedVersion(application, version, first, rest); Stream.concat(Stream.of(first), Stream.of(rest)).forEach(zone -> { for (Node node : listNodes(zone, application)) { - nodeRepository().putByHostname( + nodeRepository().putNodes( zone.getId(), new Node.Builder(node).currentVersion(node.wantedVersion()).build()); } @@ -326,7 +326,7 @@ public class SystemUpgraderTest { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); - nodeRepository().putByHostname( + nodeRepository().putNodes( zone.getId(), new Node.Builder(node).state(Node.State.failed).build()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java index e585547e921..efdf412ab77 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java @@ -14,6 +14,7 @@ import org.junit.Test; import java.time.Instant; import java.util.List; +import java.util.Optional; import static org.junit.Assert.assertEquals; @@ -23,18 +24,18 @@ import static org.junit.Assert.assertEquals; public class OsVersionStatusSerializerTest { @Test - public void test_serialization() { + public void serialization() { Version version1 = Version.fromString("7.1"); Version version2 = Version.fromString("7.2"); var versions = ImmutableMap.<OsVersion, NodeVersions>builder(); - versions.put(new OsVersion(version1, CloudName.defaultName()), NodeVersions.EMPTY.with(List.of( - new NodeVersion(HostName.from("node1"), ZoneId.from("prod", "us-west"), version1, version2, Instant.ofEpochMilli(1)), - new NodeVersion(HostName.from("node2"), ZoneId.from("prod", "us-east"), version1, version2, Instant.ofEpochMilli(2)) + versions.put(new OsVersion(version1, CloudName.defaultName()), NodeVersions.copyOf(List.of( + new NodeVersion(HostName.from("node1"), ZoneId.from("prod", "us-west"), version1, version2, Optional.of(Instant.ofEpochMilli(11))), + new NodeVersion(HostName.from("node2"), ZoneId.from("prod", "us-east"), version1, version2, Optional.of(Instant.ofEpochMilli(22))) ))); - versions.put(new OsVersion(version2, CloudName.defaultName()), NodeVersions.EMPTY.with(List.of( - new NodeVersion(HostName.from("node3"), ZoneId.from("prod", "us-west"), version2, version2, Instant.ofEpochMilli(3)), - new NodeVersion(HostName.from("node4"), ZoneId.from("prod", "us-east"), version2, version2, Instant.ofEpochMilli(4)) + versions.put(new OsVersion(version2, CloudName.defaultName()), NodeVersions.copyOf(List.of( + new NodeVersion(HostName.from("node3"), ZoneId.from("prod", "us-west"), version2, version2, Optional.of(Instant.ofEpochMilli(33))), + new NodeVersion(HostName.from("node4"), ZoneId.from("prod", "us-east"), version2, version2, Optional.of(Instant.ofEpochMilli(44))) ))); OsVersionStatusSerializer serializer = new OsVersionStatusSerializer(new OsVersionSerializer(), new NodeVersionSerializer()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index 0c1f94d90cf..41d59239321 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -2,10 +2,8 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics; import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.NodeVersions; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; @@ -14,8 +12,8 @@ import org.junit.Test; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.Optional; import static java.time.temporal.ChronoUnit.MILLIS; import static org.junit.Assert.assertEquals; @@ -31,10 +29,10 @@ public class VersionStatusSerializerTest { Version version = Version.fromString("5.0"); vespaVersions.add(new VespaVersion(version, "dead", Instant.now(), false, false, true, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"), - Instant.ofEpochMilli(123), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); vespaVersions.add(new VespaVersion(version, "cafe", Instant.now(), true, true, false, nodeVersions(Version.fromString("5.0"), Version.fromString("5.1"), - Instant.ofEpochMilli(456), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); VersionStatus status = new VersionStatus(vespaVersions); VersionStatusSerializer serializer = new VersionStatusSerializer(new NodeVersionSerializer()); VersionStatus deserialized = serializer.fromSlime(serializer.toSlime(status)); @@ -55,12 +53,12 @@ public class VersionStatusSerializerTest { } - private static NodeVersions nodeVersions(Version version, Version wantedVersion, Instant changedAt, String... hostnames) { + private static NodeVersions nodeVersions(Version version, Version wantedVersion, String... hostnames) { var nodeVersions = new ArrayList<NodeVersion>(); for (var hostname : hostnames) { - nodeVersions.add(new NodeVersion(HostName.from(hostname), ZoneId.from("prod", "us-north-1"), version, wantedVersion, changedAt)); + nodeVersions.add(new NodeVersion(HostName.from(hostname), ZoneId.from("prod", "us-north-1"), version, wantedVersion, Optional.empty())); } - return NodeVersions.EMPTY.with(nodeVersions); + return NodeVersions.copyOf(nodeVersions); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json index 3a54ac70b0d..50799bcefa2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1-log-first-part.json @@ -38,7 +38,7 @@ { "at": 0, "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": 0, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index ba51471d467..a941b824b53 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -28,7 +28,7 @@ { "at": 7303000, "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": 7303000, @@ -43,7 +43,7 @@ { "at": 7303000, "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": 7303000, @@ -58,7 +58,7 @@ { "at": 7303000, "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": 7303000, @@ -102,7 +102,7 @@ { "at": 7303000, "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": 7303000, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json index 489d6a11b6a..ec45ae02487 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-details.json @@ -33,7 +33,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -48,7 +48,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -63,7 +63,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -78,7 +78,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -122,7 +122,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -137,7 +137,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -152,7 +152,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -167,7 +167,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -182,7 +182,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", @@ -197,7 +197,7 @@ { "at": "(ignore)", "type": "info", - "message": "--- platform 6.1" + "message": "--- platform dockerImage:6.1" }, { "at": "(ignore)", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index c8036676fa9..32a77137182 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -19,9 +19,9 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Test; import java.io.File; -import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Optional; /** * @author bratseth @@ -87,8 +87,8 @@ public class DeploymentApiTest extends ControllerContainerTest { version.isControllerVersion(), version.isSystemVersion(), version.isReleased(), - NodeVersions.EMPTY.with(List.of(new NodeVersion(HostName.from("config1.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Instant.EPOCH), - new NodeVersion(HostName.from("config2.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Instant.EPOCH))), + NodeVersions.copyOf(List.of(new NodeVersion(HostName.from("config1.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Optional.empty()), + new NodeVersion(HostName.from("config2.test"), ZoneId.defaultId(), version.versionNumber(), version.versionNumber(), Optional.empty()))), version.confidence() ); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index 7370cf12d5a..9f63c714786 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -144,7 +144,7 @@ public class OsApiTest extends ControllerContainerTest { var targetVersion = nodeRepository().targetVersionsOf(zone).osVersion(application.nodeType()); for (Node node : nodeRepository().list(zone, application.id())) { var version = targetVersion.orElse(node.wantedOsVersion()); - nodeRepository().putByHostname(zone, new Node.Builder(node).currentOsVersion(version).wantedOsVersion(version).build()); + nodeRepository().putNodes(zone, new Node.Builder(node).currentOsVersion(version).wantedOsVersion(version).build()); } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index f60d11693d8..44e172d9f93 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -72,7 +72,7 @@ public class VersionStatusTest { for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) { for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { Node upgradedNode = new Node.Builder(node).currentVersion(version1).build(); - tester.configServer().nodeRepository().putByHostname(zone.getId(), upgradedNode); + tester.configServer().nodeRepository().putNodes(zone.getId(), upgradedNode); break; } } @@ -116,7 +116,7 @@ public class VersionStatusTest { for (ZoneApi zone : tester.controller().zoneRegistry().zones().all().zones()) { for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { Node downgradedNode = new Node.Builder(node).currentVersion(ancientVersion).build(); - tester.configServer().nodeRepository().putByHostname(zone.getId(), downgradedNode); + tester.configServer().nodeRepository().putNodes(zone.getId(), downgradedNode); break; } } diff --git a/default_build_settings.cmake b/default_build_settings.cmake index c74a470ae43..67d2be5917d 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -8,16 +8,17 @@ function(setup_vespa_default_build_settings_rhel_6_10) set(DEFAULT_CMAKE_SHARED_LINKER_FLAGS "-lrt" PARENT_SCOPE) endfunction() -function(setup_vespa_default_build_settings_rhel_7_7) - message("-- Setting up default build settings for rhel 7.7") +function(setup_vespa_default_build_settings_rhel_7) + message("-- Setting up default build settings for rhel 7") set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm7.0/lib" PARENT_SCOPE) set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/llvm7.0" "/usr/include/openblas" PARENT_SCOPE) set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) endfunction() -function(setup_vespa_default_build_settings_rhel_8_1) - message("-- Setting up default build settings for rhel 8.1") - set(DEFAULT_VESPA_LLVM_VERSION "7" PARENT_SCOPE) +function(setup_vespa_default_build_settings_rhel_8) + message("-- Setting up default build settings for rhel 8") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "8" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_centos_7) @@ -88,6 +89,13 @@ function(setup_vespa_default_build_settings_ubuntu_19_10) set(DEFAULT_VESPA_LLVM_VERSION "9" PARENT_SCOPE) endfunction() +function(setup_vespa_default_build_settings_ubuntu_20_04) + message("-- Setting up default build settings for ubuntu 20.04") + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/lib/llvm-10/lib" PARENT_SCOPE) + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/lib/llvm-10/include" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "10" PARENT_SCOPE) +endfunction() + function(vespa_use_default_vespa_unprivileged) if(NOT DEFINED VESPA_UNPRIVILEGED) message("-- Setting VESPA_UNPRIVILEGED to yes") @@ -152,9 +160,9 @@ function(vespa_use_default_build_settings) if(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 6.10") setup_vespa_default_build_settings_rhel_6_10() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 7.7") - setup_vespa_default_build_settings_rhel_7_7() + setup_vespa_default_build_settings_rhel_7() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rhel 8.1") - setup_vespa_default_build_settings_rhel_8_1() + setup_vespa_default_build_settings_rhel_8() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 7") setup_vespa_default_build_settings_centos_7() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 8") @@ -171,6 +179,8 @@ function(vespa_use_default_build_settings) setup_vespa_default_build_settings_fedora_33() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 19.10") setup_vespa_default_build_settings_ubuntu_19_10() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "ubuntu 20.04") + setup_vespa_default_build_settings_ubuntu_20_04() else() message(FATAL_ERROR "-- Unkonwn vespa build platform ${VESPA_OS_DISTRO_COMBINED}") endif() diff --git a/dist/vespa.spec b/dist/vespa.spec index 26310c19750..aa0232fa421 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -38,9 +38,10 @@ BuildRequires: rh-maven35 %define _rhmaven35_enable /opt/rh/rh-maven35/enable %endif %if 0%{?el8} -BuildRequires: gcc-c++ -BuildRequires: libatomic +BuildRequires: gcc-toolset-9-gcc-c++ +BuildRequires: gcc-toolset-9-binutils BuildRequires: maven +%define _devtoolset_enable /opt/rh/gcc-toolset-9/enable %endif %if 0%{?fedora} BuildRequires: gcc-c++ diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index a25a15f58ca..eb795de89b6 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -22,7 +22,7 @@ import com.github.dockerjava.core.command.PullImageResultCallback; import com.github.dockerjava.jaxrs.JerseyDockerCmdExecFactory; import com.google.inject.Inject; import com.yahoo.config.provision.DockerImage; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.dockerapi.exception.ContainerNotFoundException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerException; import com.yahoo.vespa.hosted.dockerapi.exception.DockerExecTimeoutException; @@ -78,7 +78,7 @@ public class DockerImpl implements Docker { scheduledPulls.add(image); - logger.log(LogLevel.INFO, "Starting download of " + image.asString()); + logger.log(Level.INFO, "Starting download of " + image.asString()); dockerClient.pullImageCmd(image.asString()).exec(new ImagePullCallback(image)); return true; @@ -345,14 +345,14 @@ public class DockerImpl implements Docker { @Override public void onError(Throwable throwable) { removeScheduledPoll(dockerImage); - logger.log(LogLevel.ERROR, "Could not download image " + dockerImage.asString(), throwable); + logger.log(Level.SEVERE, "Could not download image " + dockerImage.asString(), throwable); } @Override public void onComplete() { if (imageIsDownloaded(dockerImage)) { - logger.log(LogLevel.INFO, "Download completed: " + dockerImage.asString()); + logger.log(Level.INFO, "Download completed: " + dockerImage.asString()); removeScheduledPoll(dockerImage); } else { numberOfDockerApiFails.increment(); diff --git a/docproc/abi-spec.json b/docproc/abi-spec.json index dee2d2172e4..116e2c9edfb 100644 --- a/docproc/abi-spec.json +++ b/docproc/abi-spec.json @@ -157,12 +157,15 @@ ], "methods": [ "public void <init>(com.yahoo.component.ComponentId)", + "public void <init>(com.yahoo.component.ComponentId, com.yahoo.docproc.CallStack, com.yahoo.document.DocumentTypeManager, int)", "public void <init>(com.yahoo.component.ComponentId, com.yahoo.docproc.CallStack, com.yahoo.document.DocumentTypeManager)", "public void <init>(java.lang.String)", + "public void deconstruct()", "public com.yahoo.document.DocumentTypeManager getDocumentTypeManager()", "public void setDocumentTypeManager(com.yahoo.document.DocumentTypeManager)", "public int getQueueSize()", "public com.yahoo.docproc.DocprocExecutor getExecutor()", + "public java.util.concurrent.ThreadPoolExecutor getThreadPoolExecutor()", "public void setInService(boolean)", "public boolean isInService()", "public boolean isAcceptingNewProcessings()", diff --git a/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java b/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java index 3e720f9e0aa..53ccbd4c325 100644 --- a/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/AbstractConcreteDocumentFactory.java @@ -35,15 +35,15 @@ public abstract class AbstractConcreteDocumentFactory extends com.yahoo.componen /** * If the FieldValue is a StructuredFieldValue it will upgrade to the concrete type - * @param field - * @param fv + * @param field to upgrade + * @param fv value to upgrade * @return fv or upgraded fv */ public FieldValue optionallyUpgrade(Field field, FieldValue fv) { return optionallyUpgrade(field.getDataType(), fv); } - @SuppressWarnings({"unchecked", "rawtypes"}) + @SuppressWarnings({"unchecked"}) private FieldValue optionallyUpgrade(DataType dataType, FieldValue fv) { if (fv instanceof StructuredFieldValue) { try { diff --git a/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java b/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java index a1b640d5eb5..c80d9a166a8 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java +++ b/docproc/src/main/java/com/yahoo/docproc/DocprocExecutor.java @@ -5,7 +5,7 @@ import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentPut; import com.yahoo.document.json.JsonWriter; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.statistics.Counter; import com.yahoo.text.Utf8; @@ -119,7 +119,7 @@ public class DocprocExecutor { //might throw exception, which is OK: progress = call.call(processing); - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { logProgress(processing, progress, call); } @@ -150,7 +150,7 @@ public class DocprocExecutor { } } message.append("]"); - log.log(LogLevel.SPAM, message.toString()); + log.log(Level.FINEST, message.toString()); } /** diff --git a/docproc/src/main/java/com/yahoo/docproc/DocprocService.java b/docproc/src/main/java/com/yahoo/docproc/DocprocService.java index 766b433c4dd..b06e6bbdf40 100644 --- a/docproc/src/main/java/com/yahoo/docproc/DocprocService.java +++ b/docproc/src/main/java/com/yahoo/docproc/DocprocService.java @@ -3,6 +3,7 @@ package com.yahoo.docproc; import com.yahoo.component.AbstractComponent; import com.yahoo.component.ComponentId; +import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.docproc.proxy.SchemaMap; import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentTypeManager; @@ -10,6 +11,8 @@ import com.yahoo.document.DocumentTypeManager; import java.util.List; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; @@ -27,6 +30,7 @@ import java.util.logging.Logger; * * @author bratseth */ +//TODO Vespa 8 This class and a lot of other in this package should not be part of PublicAPI public class DocprocService extends AbstractComponent { private static Logger log = Logger.getLogger(DocprocService.class.getName()); @@ -34,6 +38,7 @@ public class DocprocService extends AbstractComponent { /** The processings currently in progress at this service */ private final LinkedBlockingQueue<Processing> queue; + private final ThreadPoolExecutor threadPool; /** The current state of this service */ private boolean inService = false; /** The current state of this service */ @@ -41,9 +46,18 @@ public class DocprocService extends AbstractComponent { public static SchemaMap schemaMap = new SchemaMap(); private DocumentTypeManager documentTypeManager = null; - public DocprocService(ComponentId id) { + private DocprocService(ComponentId id, int numThreads) { super(id); queue = new LinkedBlockingQueue<>(); + threadPool = new ThreadPoolExecutor(numThreads, + numThreads, + 0, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), + new DaemonThreadFactory("docproc-" + id.stringValue() + "-")); + } + + public DocprocService(ComponentId id) { + this(id, Runtime.getRuntime().availableProcessors()); } /** @@ -52,20 +66,32 @@ public class DocprocService extends AbstractComponent { * @param id the component id of the new service. * @param stack the call stack to use. * @param mgr the document type manager to use. + * @param numThreads to have in the thread pool */ - public DocprocService(ComponentId id, CallStack stack, DocumentTypeManager mgr) { - this(id); + public DocprocService(ComponentId id, CallStack stack, DocumentTypeManager mgr, int numThreads) { + this(id, numThreads); setCallStack(stack); setDocumentTypeManager(mgr); setInService(true); } + @Deprecated + public DocprocService(ComponentId id, CallStack stack, DocumentTypeManager mgr) { + this(id, stack, mgr, Runtime.getRuntime().availableProcessors()); + } + /** * Creates a service with a name with an unbounded input queue. If the given name is null or the empty string, * it will become the name "default". + * Testing only */ public DocprocService(String name) { - this(new ComponentId(name, null)); + this(new ComponentId(name, null), 1); + } + + @Override + public void deconstruct() { + threadPool.shutdown(); } public DocumentTypeManager getDocumentTypeManager() { @@ -90,6 +116,10 @@ public class DocprocService extends AbstractComponent { return executor; } + public ThreadPoolExecutor getThreadPoolExecutor() { + return threadPool; + } + private void setExecutor(DocprocExecutor executor) { this.executor = executor; } diff --git a/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java index e1bcf9c7c42..356df40af88 100644 --- a/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/SimpleDocumentProcessor.java @@ -5,7 +5,7 @@ import com.yahoo.document.DocumentOperation; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentRemove; import com.yahoo.document.DocumentUpdate; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; /** * <p>Simple layer on top of {@link DocumentProcessor}, in order to make docproc @@ -34,8 +34,8 @@ public class SimpleDocumentProcessor extends DocumentProcessor { * @param put the DocumentPut to process. */ public void process(DocumentPut put) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Ignored " + put); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Ignored " + put); } } @@ -47,8 +47,8 @@ public class SimpleDocumentProcessor extends DocumentProcessor { * @param update the DocumentUpdate to process. */ public void process(DocumentUpdate update) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Ignored " + update); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Ignored " + update); } } @@ -60,8 +60,8 @@ public class SimpleDocumentProcessor extends DocumentProcessor { * @param remove the DocumentRemove to process. */ public void process(DocumentRemove remove) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Ignored " + remove); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Ignored " + remove); } } @@ -96,8 +96,8 @@ public class SimpleDocumentProcessor extends DocumentProcessor { process((DocumentRemove) op); } } catch (RuntimeException e) { - if (log.isLoggable(LogLevel.DEBUG) && initialSize != 1) { - log.log(LogLevel.DEBUG, + if (log.isLoggable(Level.FINE) && initialSize != 1) { + log.log(Level.FINE, "Processing of document failed, from processing.getDocumentOperations() containing " + initialSize + " DocumentOperation(s).", e); } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index d34c300b8d0..335fec8c5c0 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -24,20 +24,15 @@ import com.yahoo.jdisc.Request; import com.yahoo.jdisc.handler.AbstractRequestHandler; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.log.LogLevel; import com.yahoo.messagebus.jdisc.MbusRequest; import com.yahoo.processing.execution.chain.ChainRegistry; import com.yahoo.statistics.Statistics; import java.util.TimerTask; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.logging.Logger; +import java.util.logging.Level; import static com.yahoo.component.chain.ChainsConfigurer.prepareChainRegistry; import static com.yahoo.component.chain.model.ChainsModelBuilder.buildFromConfig; @@ -53,7 +48,6 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { private final ComponentRegistry<DocprocService> docprocServiceRegistry; private final ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry; private final ChainRegistry<DocumentProcessor> chainRegistry = new ChainRegistry<>(); - private ThreadPoolExecutor threadPool; private final ScheduledThreadPoolExecutor laterExecutor = new ScheduledThreadPoolExecutor(2, new DaemonThreadFactory("docproc-later-")); private ContainerDocumentConfig containerDocConfig; @@ -62,17 +56,16 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { public DocumentProcessingHandler(ComponentRegistry<DocprocService> docprocServiceRegistry, ComponentRegistry<DocumentProcessor> documentProcessorComponentRegistry, ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry, - ThreadPoolExecutor threadPool, DocumentTypeManager documentTypeManager, + int numThreads, + DocumentTypeManager documentTypeManager, ChainsModel chainsModel, SchemaMap schemaMap, Statistics statistics, Metric metric, ContainerDocumentConfig containerDocConfig) { this.docprocServiceRegistry = docprocServiceRegistry; this.docFactoryRegistry = docFactoryRegistry; - this.threadPool = threadPool; this.containerDocConfig = containerDocConfig; this.documentTypeManager = documentTypeManager; DocprocService.schemaMap = schemaMap; - threadPool.prestartCoreThread(); laterExecutor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); laterExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); @@ -81,12 +74,12 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { for (Chain<DocumentProcessor> chain : chainRegistry.allComponents()) { log.config("Setting up call stack for chain " + chain.getId()); - DocprocService service = - new DocprocService(chain.getId(), convertToCallStack(chain, statistics, metric), documentTypeManager); + DocprocService service = new DocprocService(chain.getId(), convertToCallStack(chain, statistics, metric), documentTypeManager, computeNumThreads(numThreads)); service.setInService(true); docprocServiceRegistry.register(service.getId(), service); } } + } private static int computeNumThreads(int maxThreads) { @@ -98,12 +91,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { ComponentRegistry<AbstractConcreteDocumentFactory> docFactoryRegistry, DocumentProcessingHandlerParameters params) { this(docprocServiceRegistry, documentProcessorComponentRegistry, docFactoryRegistry, - new ThreadPoolExecutor(computeNumThreads(params.getMaxNumThreads()), - computeNumThreads(params.getMaxNumThreads()), - 0,TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - new DaemonThreadFactory("docproc-") - ), + params.getMaxNumThreads(), params.getDocumentTypeManager(), params.getChainsModel(), params.getSchemaMap(), params.getStatisticsManager(), params.getMetric(), @@ -133,7 +121,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { @Override protected void destroy() { - threadPool.shutdown(); //calling shutdownNow() seems like a bit of an overkill + //threadPoolMap.values().forEach( pool -> pool.shutdown()); //calling shutdownNow() seems like a bit of an overkill } public ComponentRegistry<DocprocService> getDocprocServiceRegistry() { @@ -175,13 +163,14 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { return null; } - DocprocService service = docprocServiceRegistry.getComponent(requestContext.getServiceName()); + String serviceName = requestContext.getServiceName(); + DocprocService service = docprocServiceRegistry.getComponent(serviceName); //No need to enqueue a task if the docproc chain is empty, just forward requestContext if (service == null) { - log.log(LogLevel.ERROR, "DocprocService for session '" + requestContext.getServiceName() + + log.log(Level.SEVERE, "DocprocService for session '" + serviceName + "' not found, returning request '" + requestContext + "'."); requestContext.processingFailed(RequestContext.ErrorCode.ERROR_PROCESSING_FAILURE, - "DocprocService " + requestContext.getServiceName() + " not found."); + "DocprocService " + serviceName + " not found."); return null; } else if (service.getExecutor().getCallStack().size() == 0) { //call stack was empty, just forward message @@ -189,19 +178,11 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { return null; } - DocumentProcessingTask task = new DocumentProcessingTask(requestContext, this, service); - submit(task); + DocumentProcessingTask task = new DocumentProcessingTask(requestContext, this, service, service.getThreadPoolExecutor()); + task.submit(); return null; } - private void submit(DocumentProcessingTask task) { - try { - threadPool.execute(task); - } catch (RejectedExecutionException ree) { - task.queueFull(); - } - } - void submit(DocumentProcessingTask task, long delay) { LaterTimerTask timerTask = new LaterTimerTask(task, delay); laterExecutor.schedule(timerTask, delay, TimeUnit.MILLISECONDS); @@ -213,14 +194,14 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { private LaterTimerTask(DocumentProcessingTask processingTask, long delay) { this.delay = delay; - log.log(LogLevel.DEBUG, "Enqueueing in " + delay + " ms due to Progress.LATER: " + processingTask); + log.log(Level.FINE, "Enqueueing in " + delay + " ms due to Progress.LATER: " + processingTask); this.processingTask = processingTask; } @Override public void run() { - log.log(LogLevel.DEBUG, "Submitting after having waited " + delay + " ms in LATER queue: " + processingTask); - submit(processingTask); + log.log(Level.FINE, "Submitting after having waited " + delay + " ms in LATER queue: " + processingTask); + processingTask.submit(); } } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java index ca4648678a5..8b330c13dff 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java @@ -9,7 +9,7 @@ import com.yahoo.docproc.DocprocService; import com.yahoo.docproc.DocumentProcessor; import com.yahoo.docproc.HandledProcessingException; import com.yahoo.docproc.Processing; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.yolean.Exceptions; import java.io.PrintWriter; @@ -17,14 +17,15 @@ import java.io.StringWriter; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; import java.util.logging.Level; import java.util.logging.Logger; /** * @author Einar M R Rosenvinge */ -public class DocumentProcessingTask implements Comparable<DocumentProcessingTask>, Runnable { +public class DocumentProcessingTask implements Runnable { private static Logger log = Logger.getLogger(DocumentProcessingTask.class.getName()); private final List<Processing> processings = new ArrayList<>(); @@ -33,16 +34,23 @@ public class DocumentProcessingTask implements Comparable<DocumentProcessingTask private final DocumentProcessingHandler docprocHandler; private RequestContext requestContext; - private final static AtomicLong seq = new AtomicLong(); - private final long seqNum; private final DocprocService service; + private final ThreadPoolExecutor executor; public DocumentProcessingTask(RequestContext requestContext, DocumentProcessingHandler docprocHandler, - DocprocService service) { - seqNum = seq.getAndIncrement(); + DocprocService service, ThreadPoolExecutor executor) { this.requestContext = requestContext; this.docprocHandler = docprocHandler; this.service = service; + this.executor = executor; + } + + void submit() { + try { + executor.execute(this); + } catch (RejectedExecutionException ree) { + queueFull(); + } } @Override @@ -52,7 +60,7 @@ public class DocumentProcessingTask implements Comparable<DocumentProcessingTask processings.addAll(requestContext.getProcessings()); } catch (Exception e) { //deserialization failed: - log.log(LogLevel.WARNING, "Deserialization of message failed.", e); + log.log(Level.WARNING, "Deserialization of message failed.", e); requestContext.processingFailed(e); return; } @@ -66,7 +74,7 @@ public class DocumentProcessingTask implements Comparable<DocumentProcessingTask } } catch (Error error) { try { - log.log(LogLevel.FATAL, Exceptions.toMessageString(error), error); + log.log(Level.SEVERE, Exceptions.toMessageString(error), error); } catch (Throwable t) { // do nothing } finally { @@ -168,34 +176,15 @@ public class DocumentProcessingTask implements Comparable<DocumentProcessingTask ". Will be automatically resent."); } - public int compareTo(DocumentProcessingTask other) { - int ourPriority = requestContext.getPriority(); - int otherPriority = other.requestContext.getPriority(); - int res = (ourPriority == otherPriority) ? 0 : ((ourPriority < otherPriority) ? -1 : 1); - if (res == 0) { - res = (seqNum == other.seqNum) ? 0 : ((seqNum < other.seqNum) ? -1 : 1); - } - return res; - } - @Override public String toString() { return "ProcessingTask{" + "processings=" + processings + ", processingsDone=" + processingsDone + ", requestContext=" + requestContext + - ", seqNum=" + seqNum + '}'; } - public int getApproxSize() { - return requestContext.getApproxSize(); - } - - final long getSeqNum() { - return seqNum; - } - private static void logProcessingFailure(Processing processing, Exception exception) { //LOGGING ONLY: String errorMsg = processing + " failed at " + processing.callStack().getLastPopped(); @@ -214,7 +203,7 @@ public class DocumentProcessingTask implements Comparable<DocumentProcessingTask if (exception != null) { StringWriter backtrace = new StringWriter(); exception.printStackTrace(new PrintWriter(backtrace)); - log.log(LogLevel.DEBUG, "Failed to process " + processing + ": " + backtrace.toString()); + log.log(Level.FINE, "Failed to process " + processing + ": " + backtrace.toString()); } } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/RequestContext.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/RequestContext.java index 8f021546ac8..48b58c12456 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/RequestContext.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/RequestContext.java @@ -21,10 +21,6 @@ public interface RequestContext { boolean isProcessable(); - int getApproxSize(); - - int getPriority(); - void processingDone(List<Processing> processing); void processingFailed(ErrorCode error, String msg); diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java index 187557791f3..2da494e19cf 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java @@ -19,7 +19,7 @@ import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.RequestDispatch; import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.jdisc.MbusRequest; @@ -47,7 +47,6 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { private final MbusRequest request; private final DocumentMessage requestMsg; private final ResponseHandler responseHandler; - private volatile int cachedApproxSize; // When spawning off new documents inside document processor, we do not want // throttling since this can lead to live locks. This is because the // document being processed is a resource and is then grabbing more resources of @@ -92,8 +91,8 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { } } } - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Forwarding " + messages.size() + " messages from " + processings.size() + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Forwarding " + messages.size() + " messages from " + processings.size() + " processings."); } if (messages.isEmpty()) { @@ -137,20 +136,6 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { } @Override - public int getApproxSize() { - if (cachedApproxSize > 0) { - return cachedApproxSize; - } - cachedApproxSize = requestMsg.getApproxSize(); - return cachedApproxSize; - } - - @Override - public int getPriority() { - return requestMsg.getPriority().getValue(); - } - - @Override public URI getUri() { return request.getUri(); } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java index 702343acd1c..92f4952b5b5 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MessageFactory.java @@ -13,7 +13,7 @@ import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.TestAndSetMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import java.util.logging.Logger; @@ -42,8 +42,8 @@ class MessageFactory { message.setTimeReceivedNow(); message.setTimeRemaining(requestMsg.getTimeRemainingNow()); message.getTrace().setLevel(requestMsg.getTrace().getLevel()); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Created '" + message.getClass().getName() + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Created '" + message.getClass().getName() + "', route = '" + message.getRoute() + "', priority = '" + message.getPriority().name() + "', load type = '" + message.getLoadType() + diff --git a/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java index 448d5213e6d..406fc9a6db9 100644 --- a/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/util/JoinerDocumentProcessor.java @@ -11,7 +11,7 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.document.datatypes.Array; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Logger; @@ -59,7 +59,7 @@ public class JoinerDocumentProcessor extends DocumentProcessor { if (op instanceof DocumentPut) { innerDocuments.add(((DocumentPut)op).getDocument()); } else { - log.log(LogLevel.DEBUG, "Skipping: " + op); + log.log(Level.FINE, "Skipping: " + op); } } processing.getDocumentOperations().clear(); diff --git a/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java b/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java index 18e4294480b..dfc36bfb641 100644 --- a/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java +++ b/docproc/src/main/java/com/yahoo/docproc/util/SplitterDocumentProcessor.java @@ -13,7 +13,7 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.document.datatypes.Array; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -42,7 +42,7 @@ public class SplitterDocumentProcessor extends DocumentProcessor { public Progress process(Processing processing) { if (processing.getDocumentOperations().size() != 1) { //we were given more than one document, return - log.log(LogLevel.DEBUG, "More than one document given, returning. (Was given " + log.log(Level.FINE, "More than one document given, returning. (Was given " + processing.getDocumentOperations().size() + " documents)."); return Progress.DONE; } @@ -57,14 +57,14 @@ public class SplitterDocumentProcessor extends DocumentProcessor { Array<Document> innerDocuments = (Array<Document>) outerDoc.getFieldValue(arrayFieldName); if (innerDocuments == null) { //the document does not have the field, return - log.log(LogLevel.DEBUG, "The given Document does not have a field value for field " + log.log(Level.FINE, "The given Document does not have a field value for field " + arrayFieldName + ", returning. (Was given " + outerDoc + ")."); return Progress.DONE; } if (innerDocuments.size() == 0) { //the array is empty, return - log.log(LogLevel.DEBUG, "The given Document does not have any elements in array field " + log.log(Level.FINE, "The given Document does not have any elements in array field " + arrayFieldName + ", returning. (Was given " + outerDoc + ")."); return Progress.DONE; } @@ -117,8 +117,8 @@ public class SplitterDocumentProcessor extends DocumentProcessor { static boolean doProcessOuterDocument(Object o, String documentTypeName) { if ( ! (o instanceof DocumentOperation)) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, o + " is not a DocumentOperation."); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, o + " is not a DocumentOperation."); } return false; } @@ -126,8 +126,8 @@ public class SplitterDocumentProcessor extends DocumentProcessor { DocumentOperation outerDocOp = (DocumentOperation)o; if ( ! (outerDocOp instanceof DocumentPut)) { //this is not a put, return - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Given DocumentOperation is not a DocumentPut, returning. (Was given " + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Given DocumentOperation is not a DocumentPut, returning. (Was given " + outerDocOp + ")."); } return false; @@ -137,8 +137,8 @@ public class SplitterDocumentProcessor extends DocumentProcessor { DocumentType type = outerDoc.getDataType(); if (!type.getName().equalsIgnoreCase(documentTypeName)) { //this is not the right document type - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Given Document is of wrong type, returning. (Was given " + outerDoc + ")."); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Given Document is of wrong type, returning. (Was given " + outerDoc + ")."); } return false; } diff --git a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingTaskPrioritizationTestCase.java b/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingTaskPrioritizationTestCase.java deleted file mode 100644 index e012239c9e3..00000000000 --- a/docproc/src/test/java/com/yahoo/docproc/jdisc/DocumentProcessingTaskPrioritizationTestCase.java +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.docproc.jdisc; - -import com.yahoo.docproc.Processing; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import org.junit.Test; - -import java.net.URI; -import java.util.List; -import java.util.Queue; -import java.util.concurrent.PriorityBlockingQueue; - -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.CoreMatchers.sameInstance; -import static org.junit.Assert.assertThat; - -/** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> - */ -public class DocumentProcessingTaskPrioritizationTestCase { - - @Test - public void proritization() { - Queue<DocumentProcessingTask> queue = new PriorityBlockingQueue<>(); - - DocumentProcessingTask highest = new TestDocumentProcessingTask(DocumentProtocol.Priority.HIGHEST); - DocumentProcessingTask veryhigh = new TestDocumentProcessingTask(DocumentProtocol.Priority.VERY_HIGH); - DocumentProcessingTask high1 = new TestDocumentProcessingTask(DocumentProtocol.Priority.HIGH_1); - DocumentProcessingTask normal_1 = new TestDocumentProcessingTask(DocumentProtocol.Priority.NORMAL_1); - DocumentProcessingTask low_1 = new TestDocumentProcessingTask(DocumentProtocol.Priority.LOW_1); - DocumentProcessingTask verylow = new TestDocumentProcessingTask(DocumentProtocol.Priority.VERY_LOW); - DocumentProcessingTask lowest = new TestDocumentProcessingTask(DocumentProtocol.Priority.LOWEST); - - DocumentProcessingTask normal_2 = new TestDocumentProcessingTask(DocumentProtocol.Priority.NORMAL_1); - DocumentProcessingTask normal_3 = new TestDocumentProcessingTask(DocumentProtocol.Priority.NORMAL_1); - DocumentProcessingTask normal_4 = new TestDocumentProcessingTask(DocumentProtocol.Priority.NORMAL_1); - - DocumentProcessingTask highest_2 = new TestDocumentProcessingTask(DocumentProtocol.Priority.HIGHEST); - DocumentProcessingTask highest_3 = new TestDocumentProcessingTask(DocumentProtocol.Priority.HIGHEST); - - - queue.add(highest); - queue.add(veryhigh); - queue.add(high1); - queue.add(normal_1); - queue.add(low_1); - queue.add(verylow); - queue.add(lowest); - - queue.add(normal_2); - queue.add(normal_3); - queue.add(normal_4); - - queue.add(highest_2); - queue.add(highest_3); - - assertThat(queue.poll(), sameInstance(highest)); - assertThat(queue.poll(), sameInstance(highest_2)); - assertThat(queue.poll(), sameInstance(highest_3)); - assertThat(queue.poll(), sameInstance(veryhigh)); - assertThat(queue.poll(), sameInstance(high1)); - assertThat(queue.poll(), sameInstance(normal_1)); - assertThat(queue.poll(), sameInstance(normal_2)); - assertThat(queue.poll(), sameInstance(normal_3)); - assertThat(queue.poll(), sameInstance(normal_4)); - assertThat(queue.poll(), sameInstance(low_1)); - assertThat(queue.poll(), sameInstance(verylow)); - assertThat(queue.poll(), sameInstance(lowest)); - assertThat(queue.poll(), nullValue()); - } - - private class TestDocumentProcessingTask extends DocumentProcessingTask { - private TestDocumentProcessingTask(DocumentProtocol.Priority priority) { - super(new TestRequestContext(priority), null, null); - } - } - - private class TestRequestContext implements RequestContext { - private final DocumentProtocol.Priority priority; - - public TestRequestContext(DocumentProtocol.Priority priority) { - this.priority = priority; - } - - @Override - public List<Processing> getProcessings() { - return null; - } - - @Override - public void skip() { - } - - @Override - public void processingDone(List<Processing> processing) { - } - - @Override - public void processingFailed(ErrorCode error, String msg) { - } - - @Override - public void processingFailed(Exception exception) { - } - - @Override - public int getApproxSize() { - return 0; - } - - @Override - public int getPriority() { - return priority.getValue(); - } - - @Override - public boolean isProcessable() { - return true; - } - - @Override - public URI getUri() { - return null; - } - - @Override - public String getServiceName() { - return null; - } - } -} diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java index aa66f6c7c3c..ec17e5e37e4 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/IndexingProcessor.java @@ -19,7 +19,7 @@ import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.indexinglanguage.AdapterFactory; import com.yahoo.vespa.indexinglanguage.SimpleAdapterFactory; @@ -89,14 +89,14 @@ public class IndexingProcessor extends DocumentProcessor { private void processDocument(DocumentPut prev, List<DocumentOperation> out) { DocumentScript script = scriptMgr.getScript(prev.getDocument().getDataType()); if (script == null) { - log.log(LogLevel.DEBUG, "No indexing script for document '%s'.", prev.getId()); + log.log(Level.FINE, "No indexing script for document '%s'.", prev.getId()); out.add(prev); return; } - log.log(LogLevel.DEBUG, "Processing document '%s'.", prev.getId()); + log.log(Level.FINE, "Processing document '%s'.", prev.getId()); Document next = script.execute(adapterFactory, prev.getDocument()); if (next == null) { - log.log(LogLevel.DEBUG, "Document '" + prev.getId() + "' produced no output."); + log.log(Level.FINE, "Document '" + prev.getId() + "' produced no output."); return; } @@ -106,14 +106,14 @@ public class IndexingProcessor extends DocumentProcessor { private void processUpdate(DocumentUpdate prev, List<DocumentOperation> out) { DocumentScript script = scriptMgr.getScript(prev.getType()); if (script == null) { - log.log(LogLevel.DEBUG, "No indexing script for update '%s'.", prev.getId()); + log.log(Level.FINE, "No indexing script for update '%s'.", prev.getId()); out.add(prev); return; } - log.log(LogLevel.DEBUG, "Processing update '%s'.", prev.getId()); + log.log(Level.FINE, "Processing update '%s'.", prev.getId()); DocumentUpdate next = script.execute(adapterFactory, prev); if (next == null) { - log.log(LogLevel.DEBUG, "Update '" + prev.getId() + "' produced no output."); + log.log(Level.FINE, "Update '" + prev.getId() + "' produced no output."); return; } next.setCondition(prev.getCondition()); @@ -121,7 +121,7 @@ public class IndexingProcessor extends DocumentProcessor { } private void processRemove(DocumentRemove prev, List<DocumentOperation> out) { - log.log(LogLevel.DEBUG, "Not processing remove '%s'.", prev.getId()); + log.log(Level.FINE, "Not processing remove '%s'.", prev.getId()); out.add(prev); } diff --git a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java index 000568214cd..a148966c250 100644 --- a/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java +++ b/docprocs/src/main/java/com/yahoo/docprocs/indexing/ScriptManager.java @@ -4,7 +4,7 @@ package com.yahoo.docprocs.indexing; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.language.Linguistics; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.indexinglanguage.ScriptParserContext; import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; @@ -35,22 +35,22 @@ public class ScriptManager { private Map<String, DocumentScript> getScripts(DocumentType inputType) { Map<String, DocumentScript> scripts = documentFieldScripts.get(inputType.getName()); if (scripts != null) { - log.log(LogLevel.DEBUG, "Using script for type '%s'.", inputType.getName()); + log.log(Level.FINE, "Using script for type '%s'.", inputType.getName()); return scripts; } for (Map.Entry<String, Map<String, DocumentScript>> entry : documentFieldScripts.entrySet()) { if (inputType.inherits(docTypeMgr.getDocumentType(entry.getKey()))) { - log.log(LogLevel.DEBUG, "Using script of super-type '%s'.", entry.getKey()); + log.log(Level.FINE, "Using script of super-type '%s'.", entry.getKey()); return entry.getValue(); } } for (Map.Entry<String, Map<String, DocumentScript>> entry : documentFieldScripts.entrySet()) { if (docTypeMgr.getDocumentType(entry.getKey()).inherits(inputType)) { - log.log(LogLevel.DEBUG, "Using script of sub-type '%s'.", entry.getKey()); + log.log(Level.FINE, "Using script of sub-type '%s'.", entry.getKey()); return entry.getValue(); } } - log.log(LogLevel.DEBUG, "No script for type '%s'.", inputType.getName()); + log.log(Level.FINE, "No script for type '%s'.", inputType.getName()); return null; } @@ -63,7 +63,7 @@ public class ScriptManager { if (fieldScripts != null) { DocumentScript script = fieldScripts.get(inputFieldName); if (script != null) { - log.log(LogLevel.DEBUG, "Using script for type '%s' and field '%s'.", inputType.getName(), inputFieldName); + log.log(Level.FINE, "Using script for type '%s' and field '%s'.", inputType.getName(), inputFieldName); return script; } } diff --git a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java index 21a163aa2b9..1d81d9e6e78 100644 --- a/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java +++ b/document/src/main/java/com/yahoo/document/DocumentTypeManagerConfigurer.java @@ -6,7 +6,7 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -72,7 +72,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub setupAnnotationTypesWithoutPayloads(config, manager); setupAnnotationRefTypes(config, manager); - log.log(LogLevel.DEBUG, "Configuring document manager with " + config.datatype().size() + " data types."); + log.log(Level.FINE, "Configuring document manager with " + config.datatype().size() + " data types."); ArrayList<DocumentmanagerConfig.Datatype> failed = new ArrayList<>(); failed.addAll(config.datatype()); while (!failed.isEmpty()) { @@ -226,7 +226,7 @@ public class DocumentTypeManagerConfigurer implements ConfigSubscriber.SingleSub DocumentTypeManager manager = configureNewManager(config); int defaultTypeCount = new DocumentTypeManager().getDataTypes().size(); if (this.managerToConfigure.getDataTypes().size() != defaultTypeCount) { - log.log(LogLevel.DEBUG, "Live document config overwritten with new config."); + log.log(Level.FINE, "Live document config overwritten with new config."); } managerToConfigure.assign(manager); } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java index 2bb93659a78..7e579a3ae6a 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/ProgressToken.java @@ -9,7 +9,7 @@ import java.util.logging.Logger; import com.yahoo.document.BucketId; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.objects.BufferSerializer; /** @@ -305,8 +305,8 @@ public class ProgressToken { && !progress.equals(FINISHED_BUCKET) && !superbucket.contains(progress) && !progress.contains(superbucket)) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "updateProgress called with non-contained bucket " + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "updateProgress called with non-contained bucket " + "pair " + superbucket + ":" + progress + ", but allowing anyway"); } } @@ -322,8 +322,8 @@ public class ProgressToken { // If progress == Integer.MAX_VALUE, we're done. Otherwise, we're not if (!progress.equals(FINISHED_BUCKET)) { if (entry.getState() != BucketState.BUCKET_ACTIVE) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "updateProgress called with sub-bucket that was " + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "updateProgress called with sub-bucket that was " + "not marked as active " + superbucket + ":" + progress); } } else { @@ -361,14 +361,14 @@ public class ProgressToken { */ protected void addBucket(BucketId superbucket, BucketId progress, BucketState state) { if (progress.equals(FINISHED_BUCKET)) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Trying to add already finished superbucket " + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Trying to add already finished superbucket " + superbucket + "; ignoring it"); } return; } - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Adding bucket pair " + superbucket + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Adding bucket pair " + superbucket + ":" + progress + " with state " + state); } @@ -789,14 +789,14 @@ public class ProgressToken { // Must not merge if sibling isn't pending if (rightSibling != null) { assert(rightSibling.getState() == BucketState.BUCKET_PENDING); - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Merging " + bucket + " with rhs " + rightCheck); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Merging " + bucket + " with rhs " + rightCheck); } // If right sibling has progress, it will unfortunately have to // be discarded if (rightSibling.getProgress().getUsedBits() != 0 - && log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Bucket progress for " + rightCheck + + && log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Bucket progress for " + rightCheck + " will be lost due to merging; potential for duplicates in result-set"); } buckets.remove(bucketToKeyWrapper(rightCheck)); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java index c26c986152c..0dd96275f9d 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/VisitorIterator.java @@ -5,7 +5,7 @@ import com.yahoo.document.BucketId; import com.yahoo.document.BucketIdFactory; import com.yahoo.document.select.BucketSelector; import com.yahoo.document.select.parser.ParseException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.Map; import java.util.Set; @@ -110,8 +110,8 @@ public class VisitorIterator { } if (!progress.isFinished()) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Importing unfinished progress token with " + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Importing unfinished progress token with " + "bits: " + progressToken.getDistributionBitCount() + ", active: " + progressToken.getActiveBucketCount() + ", pending: " + progressToken.getPendingBucketCount() + @@ -122,8 +122,8 @@ public class VisitorIterator { if (!progress.isEmpty()) { // Lower all active to pending if (progressToken.getActiveBucketCount() > 0) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Progress token had active buckets upon range " + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Progress token had active buckets upon range " + "construction. Setting these as pending"); } progressToken.setAllBucketsToState(ProgressToken.BucketState.BUCKET_PENDING); @@ -136,8 +136,8 @@ public class VisitorIterator { // Fixup for bucket cursor in case of bucket space downscaling correctTruncatedBucketCursor(); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Partial bucket space progress; continuing "+ + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Partial bucket space progress; continuing "+ "from position " + progressToken.getBucketCursor()); } } @@ -195,8 +195,8 @@ public class VisitorIterator { // half a million splits to cover the same bucket space as that 1 // single-bit bucket once did if (isLosslessResetPossible()) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "At start of bucket space and all " + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "At start of bucket space and all " + "buckets have no progress; doing a lossless reset " + "instead of splitting/merging"); } @@ -245,8 +245,8 @@ public class VisitorIterator { BucketId rightCheck = new BucketId(lastMergedBucket.getUsedBits(), lastMergedBucket.getId() | (1L << (lastMergedBucket.getUsedBits() - 1))); if (pending.equals(rightCheck)) { - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Skipped " + pending + + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Skipped " + pending + ", as it was right sibling of " + lastMergedBucket); } continue; @@ -262,8 +262,8 @@ public class VisitorIterator { } } } - if ((bucketsSplit > 0 || bucketsMerged > 0) && log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Existing progress' pending buckets had inconsistent " + + if ((bucketsSplit > 0 || bucketsMerged > 0) && log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Existing progress' pending buckets had inconsistent " + "distribution bits; performed " + bucketsSplit + " split ops and " + bucketsMerged + " merge ops. Pending: " + pendingBefore + " -> " + p.getPendingBucketCount()); @@ -285,8 +285,8 @@ public class VisitorIterator { progressToken.setBucketCursor(idx + 1); } } - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "New range bucket cursor is " + + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "New range bucket cursor is " + progressToken.getBucketCursor()); } } @@ -334,8 +334,8 @@ public class VisitorIterator { // don't do anything at all yet with the set of pending if (progressToken.getActiveBucketCount() > 0) { flushActive = true; - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Holding off new/pending buckets and consistency " + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Holding off new/pending buckets and consistency " + "correction until all " + progress.getActiveBucketCount() + " active buckets have been updated"); } @@ -349,8 +349,8 @@ public class VisitorIterator { // reset-checking to be performed correctInconsistentPending(distributionBitCount); if (delta > 0) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Increasing distribution bits for full bucket " + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Increasing distribution bits for full bucket " + "space range source from " + progressToken.getDistributionBitCount() + " to " + distributionBitCount); } @@ -359,8 +359,8 @@ public class VisitorIterator { // we go from eg. 3:0x02 to 4:0x02 to 5:02 etc. progressToken.setBucketCursor(progressToken.getBucketCursor() << delta); } else if (delta < 0) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Decreasing distribution bits for full bucket " + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Decreasing distribution bits for full bucket " + "space range source from " + progressToken.getDistributionBitCount() + " to " + distributionBitCount + " bits"); } @@ -386,15 +386,15 @@ public class VisitorIterator { // We should now always flush active buckets before doing a // consistency fix. This simplifies things greatly assert(flushActive); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received non-finished bucket " + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Received non-finished bucket " + superbucket + " with wrong distribution bit count (" + superbucket.getUsedBits() + "). Waiting to correct " + "until all active are done"); } } else { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received finished bucket " + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Received finished bucket " + superbucket + " with wrong distribution bit count (" + superbucket.getUsedBits() + "). Waiting to correct " + "until all active are done"); @@ -404,8 +404,8 @@ public class VisitorIterator { if (progressToken.getActiveBucketCount() == 0) { if (flushActive) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "All active buckets flushed, " + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "All active buckets flushed, " + "correcting progress token and continuing normal operation"); } // Trigger the actual bucket state change this time @@ -508,8 +508,8 @@ public class VisitorIterator { // a no-op, since its buckets already are fixed at 32 used bits. progress.setDistributionBitCount(distributionBitCount); this.distributionBitCount = distributionBitCount; - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Set distribution bit count to " + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Set distribution bit count to " + distributionBitCount + " for explicit bucket source (no-op)"); } } @@ -721,8 +721,8 @@ public class VisitorIterator { if (distributionBitCount != distBits) { bucketSource.setDistributionBitCount(distBits, progressToken); distributionBitCount = distBits; - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Set visitor iterator distribution bit count to " + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Set visitor iterator distribution bit count to " + distBits); } } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java index 33286d227bf..493871d45c0 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusAsyncSession.java @@ -23,7 +23,7 @@ import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentReply; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentReply; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.ErrorCode; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.MessageBus; @@ -317,7 +317,7 @@ public class MessageBusAsyncSession implements MessageBusSession, AsyncSession { @Override public void handleReply(Reply reply) { if (reply.getTrace().getLevel() > 0) { - log.log(LogLevel.INFO, reply.getTrace().toString()); + log.log(Level.INFO, reply.getTrace().toString()); } Response response = toResponse(reply); if (handler != null) { diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java index 7813cb30d7e..92abd8c98ca 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorDestinationSession.java @@ -6,7 +6,7 @@ import com.yahoo.documentapi.VisitorDestinationParameters; import com.yahoo.documentapi.VisitorDestinationSession; import com.yahoo.documentapi.VisitorResponse; import com.yahoo.documentapi.messagebus.protocol.*; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.*; import java.util.logging.Logger; @@ -56,7 +56,7 @@ public class MessageBusVisitorDestinationSession implements VisitorDestinationSe public void ack(AckToken token) { try { - log.log(LogLevel.DEBUG, "Sending ack " + token.ackObject); + log.log(Level.FINE, "Sending ack " + token.ackObject); session.reply((Reply) token.ackObject); } catch (Exception e) { e.printStackTrace(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java index 54e6fbf8be1..1ad84cb0b25 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusVisitorSession.java @@ -18,7 +18,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.documentapi.messagebus.protocol.VisitorInfoMessage; import com.yahoo.documentapi.messagebus.protocol.WrongDistributionReply; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.DestinationSession; import com.yahoo.messagebus.DestinationSessionParams; import com.yahoo.messagebus.DynamicThrottlePolicy; @@ -405,7 +405,7 @@ public class MessageBusVisitorSession implements VisitorSession { synchronized (progress.getToken()) { this.startTimeNanos = clock.monotonicNanoTime(); if (progress.getIterator().isDone()) { - log.log(LogLevel.DEBUG, sessionName + ": progress token indicates " + + log.log(Level.FINE, sessionName + ": progress token indicates " + "session is done before it could even start; no-op"); return; } @@ -433,7 +433,7 @@ public class MessageBusVisitorSession implements VisitorSession { * successful, will be equal to newState. */ private StateDescription transitionTo(StateDescription newState) { - log.log(LogLevel.DEBUG, sessionName + ": attempting transition to state " + newState); + log.log(Level.FINE, sessionName + ": attempting transition to state " + newState); switch (newState.getState()) { case WORKING: assert(state.getState() == State.NOT_STARTED); @@ -450,7 +450,7 @@ public class MessageBusVisitorSession implements VisitorSession { default: com.yahoo.protect.Process.logAndDie("Invalid target transition state: " + newState); } - log.log(LogLevel.DEBUG, "Session '" + sessionName + "' is now in state " + state); + log.log(Level.FINE, "Session '" + sessionName + "' is now in state " + state); return state; } @@ -467,7 +467,7 @@ public class MessageBusVisitorSession implements VisitorSession { // we just immediately go into a failure destruction mode as soon as this // happens, in which we do not wait for any active messages to be replied // to. - log.log(LogLevel.WARNING, "Visitor session '" + sessionName + + log.log(Level.WARNING, "Visitor session '" + sessionName + "': failed to submit reply task to executor service! " + "Session cannot reliably continue; terminating it early.", e); @@ -500,7 +500,7 @@ public class MessageBusVisitorSession implements VisitorSession { // If no cluster route has been set by user arguments, attempt to retrieve it from mbus config. if (params.getRoute() == null || !params.getRoute().hasHops()) { params.setRoute(getClusterRoute(routingTable)); - log.log(LogLevel.DEBUG, "No route specified; resolved implicit " + + log.log(Level.FINE, "No route specified; resolved implicit " + "storage cluster: " + params.getRoute().toString()); } } @@ -574,8 +574,8 @@ public class MessageBusVisitorSession implements VisitorSession { 1, progressToken); } else { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "parameters specify explicit bucket set " + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "parameters specify explicit bucket set " + "to visit; using it rather than document selection (" + params.getBucketsToVisit().size() + " buckets given)"); } @@ -647,7 +647,7 @@ public class MessageBusVisitorSession implements VisitorSession { VisitorIterator.BucketProgress bucket = progress.getIterator().getNext(); Result result = sender.send(createMessage(bucket)); if (result.isAccepted()) { - log.log(LogLevel.DEBUG, sessionName + ": sent CreateVisitor for bucket " + + log.log(Level.FINE, sessionName + ": sent CreateVisitor for bucket " + bucket.getSuperbucket() + " with progress " + bucket.getProgress()); ++pendingMessageCount; } else { @@ -661,7 +661,7 @@ public class MessageBusVisitorSession implements VisitorSession { String msg = "Got exception of type " + e.getClass().getName() + " with message '" + e.getMessage() + "' while attempting to send visitors"; - log.log(LogLevel.WARNING, msg); + log.log(Level.WARNING, msg); transitionTo(new StateDescription(State.FAILED, msg)); // It's likely that the exception caused a failure to send a // visitor message, meaning we won't get a reply task in the @@ -688,7 +688,7 @@ public class MessageBusVisitorSession implements VisitorSession { // 'done' is only ever written when token mutex is held, so safe to check // outside of completionMonitor lock. assert(!done) : "Session was marked as completed more than once"; - log.log(LogLevel.DEBUG, "Visitor session '" + sessionName + "' has completed"); + log.log(Level.FINE, "Visitor session '" + sessionName + "' has completed"); if (params.getLocalDataHandler() != null) { params.getLocalDataHandler().onDone(); } @@ -726,7 +726,7 @@ public class MessageBusVisitorSession implements VisitorSession { } else { String msg = "Received reply we do not know how to handle: " + reply.getClass().getName(); - log.log(LogLevel.ERROR, msg); + log.log(Level.SEVERE, msg); transitionTo(new StateDescription(State.FAILED, msg)); } } catch (Exception e) { @@ -754,8 +754,8 @@ public class MessageBusVisitorSession implements VisitorSession { @Override public void run() { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Visitor session " + sessionName + ": Received message " + message); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Visitor session " + sessionName + ": Received message " + message); } try { if (message instanceof VisitorInfoMessage) { @@ -772,7 +772,7 @@ public class MessageBusVisitorSession implements VisitorSession { private void handleMessageProcessingException(Reply reply, Exception e, String what) { final String errorDesc = formatProcessingException(e, what); final String fullMsg = formatIdentifyingVisitorErrorString(errorDesc); - log.log(LogLevel.ERROR, fullMsg, e); + log.log(Level.SEVERE, fullMsg, e); int errorCode; synchronized (progress.getToken()) { if (!params.skipBucketsOnFatalErrors()) { @@ -809,8 +809,8 @@ public class MessageBusVisitorSession implements VisitorSession { Reply reply = msg.createReply(); msg.swapState(reply); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Visitor session " + sessionName + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Visitor session " + sessionName + ": Received VisitorInfo with " + msg.getFinishedBuckets().size() + " finished buckets"); } @@ -841,7 +841,7 @@ public class MessageBusVisitorSession implements VisitorSession { msg.swapState(reply); if (params.getLocalDataHandler() == null) { - log.log(LogLevel.ERROR, sessionName + ": Got visitor data back to client with no local data destination."); + log.log(Level.SEVERE, sessionName + ": Got visitor data back to client with no local data destination."); reply.addError(new Error(ErrorCode.APP_FATAL_ERROR, "Visitor data with no local data destination")); receiver.reply(reply); return; @@ -902,7 +902,7 @@ public class MessageBusVisitorSession implements VisitorSession { progress.getIterator().update(bucket, subProgress); String message = getErrorMessage(reply.getError(0)); - log.log(LogLevel.DEBUG, sessionName + ": received error reply for bucket " + + log.log(Level.FINE, sessionName + ": received error reply for bucket " + bucket + " with message '" + message + "'"); if (isFatalError(reply)) { @@ -1027,7 +1027,7 @@ public class MessageBusVisitorSession implements VisitorSession { BucketId superbucket = msg.getBuckets().get(0); BucketId subBucketProgress = reply.getLastBucket(); - log.log(LogLevel.DEBUG, sessionName + ": received CreateVisitorReply for bucket " + + log.log(Level.FINE, sessionName + ": received CreateVisitorReply for bucket " + superbucket + " with progress " + subBucketProgress); progress.getIterator().update(superbucket, subBucketProgress); @@ -1045,7 +1045,7 @@ public class MessageBusVisitorSession implements VisitorSession { int newMaxBuckets = Math.max(Math.min((int)(params.getMaxBucketsPerVisitor() * params.getDynamicMaxBucketsIncreaseFactor()), 128), 1); params.setMaxBucketsPerVisitor(newMaxBuckets); - log.log(LogLevel.DEBUG, sessionName + ": increasing max buckets per visitor to " + log.log(Level.FINE, sessionName + ": increasing max buckets per visitor to " + params.getMaxBucketsPerVisitor()); } } @@ -1055,7 +1055,7 @@ public class MessageBusVisitorSession implements VisitorSession { ClusterState newState = new ClusterState(reply.getSystemState()); int stateBits = newState.getDistributionBitCount(); if (stateBits != progress.getIterator().getDistributionBitCount()) { - log.log(LogLevel.DEBUG, "System state changed; now at " + + log.log(Level.FINE, "System state changed; now at " + stateBits + " distribution bits"); // Update the internal state of the visitor iterator. If we're increasing // the number of distribution bits, this may lead to splitting of pending @@ -1066,7 +1066,7 @@ public class MessageBusVisitorSession implements VisitorSession { progress.getIterator().setDistributionBitCount(stateBits); } } catch (Exception e) { - log.log(LogLevel.ERROR, "Failed to parse new system state string: " + log.log(Level.SEVERE, "Failed to parse new system state string: " + reply.getSystemState()); transitionTo(new StateDescription(State.FAILED, "Failed to parse cluster state '" + reply.getSystemState() + "'")); @@ -1101,8 +1101,8 @@ public class MessageBusVisitorSession implements VisitorSession { @Override public void ack(AckToken token) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Visitor session " + sessionName + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Visitor session " + sessionName + ": Sending ack " + token.ackObject); } // No locking here; replying should be thread safe in itself @@ -1144,7 +1144,7 @@ public class MessageBusVisitorSession implements VisitorSession { @Override public void destroy() { - log.log(LogLevel.DEBUG, sessionName + ": synchronous destroy() called"); + log.log(Level.FINE, sessionName + ": synchronous destroy() called"); try { synchronized (progress.getToken()) { synchronized (completionMonitor) { @@ -1170,9 +1170,9 @@ public class MessageBusVisitorSession implements VisitorSession { sender.destroy(); receiver.destroy(); } catch (Exception e) { - log.log(LogLevel.ERROR, "Caught exception destroying communication interfaces", e); + log.log(Level.SEVERE, "Caught exception destroying communication interfaces", e); } - log.log(LogLevel.DEBUG, sessionName + ": synchronous destroy() done"); + log.log(Level.FINE, sessionName + ": synchronous destroy() done"); } } } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java index cd181ac3c51..8fbd1548f68 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/DocumentRouteSelectorPolicy.java @@ -5,7 +5,7 @@ import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.document.DocumentGet; import com.yahoo.document.select.DocumentSelector; import com.yahoo.document.select.Result; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingContext; @@ -68,7 +68,7 @@ public class DocumentRouteSelectorPolicy DocumentSelector selector; try { selector = new DocumentSelector(route.selector()); - log.log(LogLevel.CONFIG, "Selector for route '" + route.name() + "' is '" + selector + "'."); + log.log(Level.CONFIG, "Selector for route '" + route.name() + "' is '" + selector + "'."); } catch (com.yahoo.document.select.parser.ParseException e) { error = "Error parsing selector '" + route.selector() + "' for route '" + route.name() + "; " + e.getMessage(); diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java index 1f3810f9a02..ace19466496 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableFactories60.java @@ -12,7 +12,7 @@ import com.yahoo.document.serialization.DocumentDeserializer; import com.yahoo.document.serialization.DocumentSerializer; import com.yahoo.document.serialization.DocumentSerializerFactory; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Routable; import com.yahoo.vdslib.DocumentSummary; import com.yahoo.vdslib.SearchResult; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java index 3933b188dc6..7fc1ad8fc84 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutableRepository.java @@ -11,7 +11,7 @@ import com.yahoo.document.serialization.DocumentSerializer; import com.yahoo.document.serialization.DocumentSerializerFactory; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Routable; import java.util.ArrayList; @@ -51,11 +51,11 @@ final class RoutableRepository { */ Routable decode(DocumentTypeManager docMan, Version version, byte[] data) { if (data == null || data.length == 0) { - log.log(LogLevel.ERROR, "Received empty byte array for deserialization."); + log.log(Level.SEVERE, "Received empty byte array for deserialization."); return null; } if (version.getMajor() < 5) { - log.log(LogLevel.ERROR,"Can not decode anything from (version " + version + "). Only major version 5 and up supported."); + log.log(Level.SEVERE,"Can not decode anything from (version " + version + "). Only major version 5 and up supported."); return null; } DocumentDeserializer in = DocumentDeserializerFactory.createHead(docMan, GrowableByteBuffer.wrap(data)); @@ -64,12 +64,12 @@ final class RoutableRepository { int type = in.getInt(null); RoutableFactory factory = getFactory(version, type); if (factory == null) { - log.log(LogLevel.ERROR,"No routable factory found for routable type " + type + " (version " + version + ")."); + log.log(Level.SEVERE,"No routable factory found for routable type " + type + " (version " + version + ")."); return null; } Routable ret = factory.decode(in, loadTypes); if (ret == null) { - log.log(LogLevel.ERROR,"Routable factory " + factory.getClass().getName() + " failed to deserialize " + + log.log(Level.SEVERE,"Routable factory " + factory.getClass().getName() + " failed to deserialize " + "routable of type " + type + " (version " + version + ").\nData = " + Arrays.toString(data)); return null; } @@ -90,18 +90,18 @@ final class RoutableRepository { int type = obj.getType(); RoutableFactory factory = getFactory(version, type); if (factory == null) { - log.log(LogLevel.ERROR,"No routable factory found for routable type " + type + " (version " + version + ")."); + log.log(Level.SEVERE,"No routable factory found for routable type " + type + " (version " + version + ")."); return new byte[0]; } if (version.getMajor() < 5) { - log.log(LogLevel.ERROR,"Can not encode routable type " + type + " (version " + version + "). Only major version 5 and up supported."); + log.log(Level.SEVERE,"Can not encode routable type " + type + " (version " + version + "). Only major version 5 and up supported."); return new byte[0]; } DocumentSerializer out= DocumentSerializerFactory.createHead(new GrowableByteBuffer(8192)); out.putInt(null, type); if (!factory.encode(obj, out)) { - log.log(LogLevel.ERROR, "Routable factory " + factory.getClass().getName() + " failed to serialize " + + log.log(Level.SEVERE, "Routable factory " + factory.getClass().getName() + " failed to serialize " + "routable of type " + type + " (version " + version + ")."); return new byte[0]; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java index 098e0efc41c..c9e3488ecb9 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/RoutingPolicyRepository.java @@ -2,7 +2,7 @@ package com.yahoo.documentapi.messagebus.protocol; import com.yahoo.messagebus.routing.RoutingPolicy; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.Map; import java.util.logging.Logger; @@ -49,13 +49,13 @@ class RoutingPolicyRepository { RoutingPolicy createPolicy(String name, String param) { RoutingPolicyFactory factory = getFactory(name); if (factory == null) { - log.log(LogLevel.ERROR, "No routing policy factory found for name '" + name + "'."); + log.log(Level.SEVERE, "No routing policy factory found for name '" + name + "'."); return null; } DocumentProtocolRoutingPolicy ret = factory.createPolicy(param); if (ret == null) { - log.log(LogLevel.ERROR, "Routing policy factory " + factory.getClass().getName() + " failed to create a " + + log.log(Level.SEVERE, "Routing policy factory " + factory.getClass().getName() + " failed to create a " + "routing policy for parameter '" + name + "'."); return null; } diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java index 46d0cd3fc37..5594b9f9db4 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/StoragePolicy.java @@ -6,7 +6,7 @@ import com.yahoo.document.BucketId; import com.yahoo.document.BucketIdFactory; import com.yahoo.jrt.slobrok.api.IMirror; import com.yahoo.jrt.slobrok.api.Mirror; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; @@ -148,7 +148,7 @@ public class StoragePolicy extends SlobrokPolicy { if (arr.size() == 1) { return convertSlobrokNameToSessionName(arr.get(0).getSpecString()); } else { - log.log(LogLevel.WARNING, "Got " + arr.size() + " matches for a distributor."); + log.log(Level.WARNING, "Got " + arr.size() + " matches for a distributor."); } } else { return convertSlobrokNameToSessionName(arr.get(randomizer.nextInt(arr.size())).getSpecString()); @@ -282,7 +282,7 @@ public class StoragePolicy extends SlobrokPolicy { case DocumentProtocol.MESSAGE_CREATEVISITOR: return ((CreateVisitorMessage)msg).getBuckets().get(0); case DocumentProtocol.MESSAGE_REMOVELOCATION: return ((RemoveLocationMessage)msg).getBucketId(); default: - log.log(LogLevel.ERROR, "Message type '" + msg.getType() + "' not supported."); + log.log(Level.SEVERE, "Message type '" + msg.getType() + "' not supported."); return null; } } @@ -395,7 +395,7 @@ public class StoragePolicy extends SlobrokPolicy { return targetSpec; } else { sendRandomReason = "Want to use distributor " + target + " but it is not in slobrok. Sending to random."; - log.log(LogLevel.DEBUG, "Target distributor is not in slobrok"); + log.log(Level.FINE, "Target distributor is not in slobrok"); } } else { context.setContext(new MessageContext(cachedClusterState)); @@ -407,7 +407,7 @@ public class StoragePolicy extends SlobrokPolicy { context.setReply(reply); return null; } catch (Distribution.NoDistributorsAvailableException e) { - log.log(LogLevel.DEBUG, "No distributors available; clearing cluster state"); + log.log(Level.FINE, "No distributors available; clearing cluster state"); safeCachedClusterState.set(null); sendRandomReason = "No distributors available. Sending to random distributor."; context.setContext(createRandomDistributorTargetContext()); @@ -471,7 +471,7 @@ public class StoragePolicy extends SlobrokPolicy { if (context.usedState == null) { String msg = "Used state must be set as distributor is calculated. Bug."; reply.getTrace().trace(1, msg); - log.log(LogLevel.ERROR, msg); + log.log(Level.SEVERE, msg); } else if (newState.getVersion() == context.usedState.getVersion()) { String msg = "Message sent to distributor " + context.calculatedDistributor + " retrieved cluster state version " + newState.getVersion() + @@ -482,7 +482,7 @@ public class StoragePolicy extends SlobrokPolicy { // reject an operation bound to a particular bucket if it does not own the bucket in _both_ // the current and the next (transition target) state. Since it can happen during normal operation // and will happen per client operation, we keep this as debug level to prevent spamming the logs. - log.log(LogLevel.DEBUG, msg); + log.log(Level.FINE, msg); } else if (newState.getVersion() > context.usedState.getVersion()) { if (reply.getTrace().shouldTrace(1)) { reply.getTrace().trace(1, "Message sent to distributor " + context.calculatedDistributor + diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java index 3251c038fc7..ee851acc0f8 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/SubsetServicePolicy.java @@ -2,7 +2,7 @@ package com.yahoo.documentapi.messagebus.protocol; import com.yahoo.jrt.slobrok.api.Mirror; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.routing.Hop; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingContext; @@ -38,7 +38,7 @@ public class SubsetServicePolicy implements DocumentProtocolRoutingPolicy { subsetSize = Integer.parseInt(param); } catch (NumberFormatException e) { - log.log(LogLevel.WARNING, "Parameter '" + param + "' could not be parsed as an integer.", e); + log.log(Level.WARNING, "Parameter '" + param + "' could not be parsed as an integer.", e); } if (subsetSize <= 0) { log.warning("Ignoring a request to set the subset size to " + subsetSize + " because it makes no " + diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java index 69a5346adb5..d23a0e580a0 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus.systemstate.rule; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; @@ -91,7 +91,7 @@ public class NodeState { } if (arr[0].equals(NODE_PARENT)) { if (parent == null) { - log.log(LogLevel.ERROR, "Location string '" + key + "' requests a parent above the top-most node, " + + log.log(Level.SEVERE, "Location string '" + key + "' requests a parent above the top-most node, " + "returning self to avoid crash."); } return parent.getChild(arr[1], force); diff --git a/eval/src/vespa/eval/eval/make_tensor_function.cpp b/eval/src/vespa/eval/eval/make_tensor_function.cpp index f503532c1f9..3a73a3b8784 100644 --- a/eval/src/vespa/eval/eval/make_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/make_tensor_function.cpp @@ -59,37 +59,37 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { void make_reduce(const Node &, Aggr aggr, const std::vector<vespalib::string> &dimensions) { assert(stack.size() >= 1); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::reduce(a, aggr, dimensions, stash); } void make_map(const Node &, map_fun_t function) { assert(stack.size() >= 1); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::map(a, function, stash); } void make_join(const Node &, join_fun_t function) { assert(stack.size() >= 2); - const auto &b = stack.back(); + const auto &b = stack.back().get(); stack.pop_back(); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::join(a, b, function, stash); } void make_merge(const Node &, join_fun_t function) { assert(stack.size() >= 2); - const auto &b = stack.back(); + const auto &b = stack.back().get(); stack.pop_back(); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::merge(a, b, function, stash); } void make_concat(const Node &, const vespalib::string &dimension) { assert(stack.size() >= 2); - const auto &b = stack.back(); + const auto &b = stack.back().get(); stack.pop_back(); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::concat(a, b, dimension, stash); } @@ -156,17 +156,17 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { void make_rename(const Node &, const std::vector<vespalib::string> &from, const std::vector<vespalib::string> &to) { assert(stack.size() >= 1); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::rename(a, from, to, stash); } void make_if(const Node &) { assert(stack.size() >= 3); - const auto &c = stack.back(); + const auto &c = stack.back().get(); stack.pop_back(); - const auto &b = stack.back(); + const auto &b = stack.back().get(); stack.pop_back(); - const auto &a = stack.back(); + const auto &a = stack.back().get(); stack.back() = tensor_function::if_node(a, b, c, stash); } diff --git a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java index bd4401d3cab..32c381c1e38 100644 --- a/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java +++ b/fileacquirer/src/main/java/com/yahoo/filedistribution/fileacquirer/FileAcquirerImpl.java @@ -5,7 +5,7 @@ import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.config.FileReference; import com.yahoo.jrt.*; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.concurrent.locks.Lock; @@ -60,7 +60,7 @@ class FileAcquirerImpl implements FileAcquirer { logWarning(); target.close(); } else { - log.log(LogLevel.DEBUG, "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); + log.log(Level.FINE, "Successfully connected to '" + spec + "', this = " + System.identityHashCode(this)); pauseTime = 0; logCount = 0; return; @@ -152,7 +152,7 @@ class FileAcquirerImpl implements FileAcquirer { request.parameters().add(new StringValue(fileReference.value())); double rpcTimeout = Math.min(timer.timeLeft(TimeUnit.SECONDS), 60.0); - log.log(LogLevel.DEBUG, "InvokeSync waitFor " + fileReference + " with " + rpcTimeout + " seconds timeout"); + log.log(Level.FINE, "InvokeSync waitFor " + fileReference + " with " + rpcTimeout + " seconds timeout"); target.invokeSync(request, rpcTimeout); if (request.checkReturnTypes("s")) { @@ -160,7 +160,7 @@ class FileAcquirerImpl implements FileAcquirer { } else if (!request.isError()) { throw new RuntimeException("Invalid response: " + request.returnValues()); } else if (temporaryError(request.errorCode())) { - log.log(LogLevel.INFO, "Retrying waitFor for " + fileReference + ": " + request.errorCode() + " -- " + request.errorMessage()); + log.log(Level.INFO, "Retrying waitFor for " + fileReference + ": " + request.errorCode() + " -- " + request.errorMessage()); Thread.sleep(1000); } else { if (request.errorCode() == FileDistributionErrorCode.fileReferenceDoesNotExists) diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index 133268d5872..8bfa85d8b16 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -30,11 +30,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>config</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java index ddab35ba14d..0ed72076549 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/CompressedFileReference.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.filedistribution; import com.google.common.io.ByteStreams; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveInputStream; import org.apache.commons.compress.archivers.ArchiveOutputStream; @@ -63,7 +63,7 @@ public class CompressedFileReference { } static void decompress(File inputFile, File outputDir) throws IOException { - log.log(LogLevel.DEBUG, () -> "Decompressing '" + inputFile + "' into '" + outputDir + "'"); + log.log(Level.FINE, () -> "Decompressing '" + inputFile + "' into '" + outputDir + "'"); try (ArchiveInputStream ais = new TarArchiveInputStream(new GZIPInputStream(new FileInputStream(inputFile)))) { decompress(ais, outputDir); } catch (IllegalArgumentException e) { @@ -78,16 +78,16 @@ public class CompressedFileReference { File outFile = new File(outputFile, entry.getName()); if (entry.isDirectory()) { if (!(outFile.exists() && outFile.isDirectory())) { - log.log(LogLevel.DEBUG, () -> "Creating dir: " + outFile.getAbsolutePath()); + log.log(Level.FINE, () -> "Creating dir: " + outFile.getAbsolutePath()); if (!outFile.mkdirs()) { - log.log(LogLevel.WARNING, "Could not create dir " + entry.getName()); + log.log(Level.WARNING, "Could not create dir " + entry.getName()); } } } else { // Create parent dir if necessary File parent = new File(outFile.getParent()); if (!parent.exists() && !parent.mkdirs()) { - log.log(LogLevel.WARNING, "Could not create dir " + parent.getAbsolutePath()); + log.log(Level.WARNING, "Could not create dir " + parent.getAbsolutePath()); } FileOutputStream fos = new FileOutputStream(outFile); ByteStreams.copy(archiveInputStream, fos); @@ -113,7 +113,7 @@ public class CompressedFileReference { } private static void writeFileToTar(ArchiveOutputStream taos, File baseDir, File file) throws IOException { - log.log(LogLevel.DEBUG, () -> "Adding file to tar: " + baseDir.toPath().relativize(file.toPath()).toString()); + log.log(Level.FINE, () -> "Adding file to tar: " + baseDir.toPath().relativize(file.toPath()).toString()); taos.putArchiveEntry(taos.createArchiveEntry(file, baseDir.toPath().relativize(file.toPath()).toString())); ByteStreams.copy(new FileInputStream(file), taos); taos.closeArchiveEntry(); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index 5829ab37b77..a68d2860874 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.filedistribution; import com.yahoo.config.FileReference; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.defaults.Defaults; @@ -51,7 +51,7 @@ public class FileDownloader { try { return getFutureFile(fileReferenceDownload).get(timeout.toMillis(), TimeUnit.MILLISECONDS); } catch (InterruptedException | ExecutionException | TimeoutException e) { - log.log(LogLevel.WARNING, "Failed downloading '" + fileReferenceDownload.fileReference().value() + "', removing from download queue: " + e.getMessage()); + log.log(Level.WARNING, "Failed downloading '" + fileReferenceDownload.fileReference().value() + "', removing from download queue: " + e.getMessage()); fileReferenceDownloader.failedDownloading(fileReferenceDownload.fileReference()); return Optional.empty(); } @@ -89,7 +89,7 @@ public class FileDownloader { } else if (!file.canRead()) { throw new RuntimeException("File reference '" + fileReference.value() + "'exists, but unable to read it"); } else { - log.log(LogLevel.DEBUG, () -> "File reference '" + fileReference.value() + "' found: " + file.getAbsolutePath()); + log.log(Level.FINE, () -> "File reference '" + fileReference.value() + "' found: " + file.getAbsolutePath()); fileReferenceDownloader.setDownloadStatus(fileReference, 1.0); return Optional.of(file); } diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index ff9efc74929..875058b2eff 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -8,7 +8,7 @@ import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Method; import com.yahoo.jrt.Request; import com.yahoo.jrt.Supervisor; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import net.jpountz.xxhash.StreamingXXHash64; import net.jpountz.xxhash.XXHashFactory; @@ -80,7 +80,7 @@ public class FileReceiver { inprogressFile = Files.createTempFile(tmpDirectory.toPath(), fileName, ".inprogress").toFile(); } catch (IOException e) { String msg = "Failed creating temp file for inprogress file for " + fileName + " in '" + tmpDirectory.toPath() + "': "; - log.log(LogLevel.ERROR, msg + e.getMessage(), e); + log.log(Level.SEVERE, msg + e.getMessage(), e); throw new RuntimeException(msg, e); } } @@ -96,7 +96,7 @@ public class FileReceiver { try { Files.write(inprogressFile.toPath(), part, StandardOpenOption.WRITE, StandardOpenOption.APPEND); } catch (IOException e) { - log.log(LogLevel.ERROR, "Failed writing to file (" + inprogressFile.toPath() + "): " + e.getMessage(), e); + log.log(Level.SEVERE, "Failed writing to file (" + inprogressFile.toPath() + "): " + e.getMessage(), e); inprogressFile.delete(); throw new RuntimeException("Failed writing to file (" + inprogressFile.toPath() + "): ", e); } @@ -120,14 +120,14 @@ public class FileReceiver { try { Files.createDirectories(fileReferenceDir.toPath()); } catch (IOException e) { - log.log(LogLevel.ERROR, "Failed creating directory (" + fileReferenceDir.toPath() + "): " + e.getMessage(), e); + log.log(Level.SEVERE, "Failed creating directory (" + fileReferenceDir.toPath() + "): " + e.getMessage(), e); throw new RuntimeException("Failed creating directory (" + fileReferenceDir.toPath() + "): ", e); } - log.log(LogLevel.DEBUG, () -> "Uncompressed file, moving to " + file.getAbsolutePath()); + log.log(Level.FINE, () -> "Uncompressed file, moving to " + file.getAbsolutePath()); moveFileToDestination(inprogressFile, file); } } catch (IOException e) { - log.log(LogLevel.ERROR, "Failed writing file: " + e.getMessage(), e); + log.log(Level.SEVERE, "Failed writing file: " + e.getMessage(), e); throw new RuntimeException("Failed writing file: ", e); } finally { try { @@ -135,7 +135,7 @@ public class FileReceiver { Files.delete(inprogressFile.toPath()); } } catch (IOException e) { - log.log(LogLevel.ERROR, "Failed deleting " + inprogressFile.getAbsolutePath() + ": " + e.getMessage(), e); + log.log(Level.SEVERE, "Failed deleting " + inprogressFile.getAbsolutePath() + ": " + e.getMessage(), e); } } return file; @@ -188,16 +188,16 @@ public class FileReceiver { private static void moveFileToDestination(File tempFile, File destination) { try { Files.move(tempFile.toPath(), destination.toPath()); - log.log(LogLevel.DEBUG, () -> "File moved from " + tempFile.getAbsolutePath()+ " to " + destination.getAbsolutePath()); + log.log(Level.FINE, () -> "File moved from " + tempFile.getAbsolutePath()+ " to " + destination.getAbsolutePath()); } catch (FileAlreadyExistsException e) { // Don't fail if it already exists (we might get the file from several config servers when retrying, servers are down etc. // so it might be written already). Delete temp file/dir in that case, to avoid filling the disk. - log.log(LogLevel.INFO, "Failed moving file '" + tempFile.getAbsolutePath() + "' to '" + destination.getAbsolutePath() + + log.log(Level.INFO, "Failed moving file '" + tempFile.getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "', '" + destination.getAbsolutePath() + "' already exists"); deleteFileOrDirectory(tempFile); } catch (IOException e) { String message = "Failed moving file '" + tempFile.getAbsolutePath() + "' to '" + destination.getAbsolutePath() + "'"; - log.log(LogLevel.ERROR, message, e); + log.log(Level.SEVERE, message, e); throw new RuntimeException(message, e); } } @@ -210,12 +210,12 @@ public class FileReceiver { else Files.delete(path.toPath()); } catch (IOException ioe) { - log.log(LogLevel.WARNING, "Failed deleting file/dir " + path); + log.log(Level.WARNING, "Failed deleting file/dir " + path); } } private void receiveFileMeta(Request req) { - log.log(LogLevel.DEBUG, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(Level.FINE, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); String fileName = req.parameters().get(1).asString(); String type = req.parameters().get(2).asString(); @@ -240,7 +240,7 @@ public class FileReceiver { } private void receiveFilePart(Request req) { - log.log(LogLevel.DEBUG, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(Level.FINE, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); int sessionId = req.parameters().get(1).asInt32(); @@ -255,13 +255,13 @@ public class FileReceiver { retval = 1; } double completeness = (double) session.currentFileSize / (double) session.fileSize; - log.log(LogLevel.SPAM, () -> String.format("%.1f percent of '%s' downloaded", completeness * 100, reference.value())); + log.log(Level.FINEST, () -> String.format("%.1f percent of '%s' downloaded", completeness * 100, reference.value())); downloader.setDownloadStatus(reference, completeness); req.returnValues().add(new Int32Value(retval)); } private void receiveFileEof(Request req) { - log.log(LogLevel.DEBUG, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); + log.log(Level.FINE, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters()); FileReference reference = new FileReference(req.parameters().get(0).asString()); int sessionId = req.parameters().get(1).asInt32(); long xxhash = req.parameters().get(2).asInt64(); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index f1d31156d79..4ee9eacc2be 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -6,7 +6,7 @@ import com.yahoo.config.FileReference; import com.yahoo.jrt.Int32Value; import com.yahoo.jrt.Request; import com.yahoo.jrt.StringValue; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.config.Connection; import com.yahoo.vespa.config.ConnectionPool; @@ -81,7 +81,7 @@ public class FileReferenceDownloader { void addToDownloadQueue(FileReferenceDownload fileReferenceDownload) { FileReference fileReference = fileReferenceDownload.fileReference(); - log.log(LogLevel.DEBUG, () -> "Will download file reference '" + fileReference.value() + "' with timeout " + downloadTimeout); + log.log(Level.FINE, () -> "Will download file reference '" + fileReference.value() + "' with timeout " + downloadTimeout); synchronized (downloads) { downloads.put(fileReference, fileReferenceDownload); downloadStatus.put(fileReference, 0.0); @@ -97,7 +97,7 @@ public class FileReferenceDownloader { downloads.remove(fileReference); download.future().complete(Optional.of(file)); } else { - log.log(LogLevel.DEBUG, () -> "Received '" + fileReference + "', which was not requested. Can be ignored if happening during upgrades/restarts"); + log.log(Level.FINE, () -> "Received '" + fileReference + "', which was not requested. Can be ignored if happening during upgrades/restarts"); } } } @@ -117,7 +117,7 @@ public class FileReferenceDownloader { request.parameters().add(new Int32Value(fileReferenceDownload.downloadFromOtherSourceIfNotFound() ? 0 : 1)); connection.invokeSync(request, (double) rpcTimeout.getSeconds()); - Level logLevel = (retryCount > 0 ? LogLevel.INFO : LogLevel.DEBUG); + Level logLevel = (retryCount > 0 ? Level.INFO : Level.FINE); if (validateResponse(request)) { log.log(logLevel, () -> "Request callback, OK. Req: " + request + "\nSpec: " + connection + ", retry count " + retryCount); if (request.returnValues().get(0).asInt32() == 0) { @@ -155,7 +155,7 @@ public class FileReferenceDownloader { } else if (request.returnValues().size() == 0) { return false; } else if (!request.checkReturnTypes("is")) { // TODO: Do not hard-code return type - log.log(LogLevel.WARNING, "Invalid return types for response: " + request.errorMessage()); + log.log(Level.WARNING, "Invalid return types for response: " + request.errorMessage()); return false; } return true; diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java index 26cc3553c1f..4e0d5e4e32a 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/RpcTester.java @@ -13,7 +13,7 @@ import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import net.jpountz.xxhash.XXHash64; import net.jpountz.xxhash.XXHashFactory; @@ -49,19 +49,19 @@ public class RpcTester { e.printStackTrace(); } - log.log(LogLevel.INFO, "Read blob from " + file.getAbsolutePath()); + log.log(Level.INFO, "Read blob from " + file.getAbsolutePath()); Supervisor supervisor = new Supervisor(new Transport()); Spec spec = new Spec("tcp/localhost:19090"); - log.log(LogLevel.INFO, "Connecting to " + spec); + log.log(Level.INFO, "Connecting to " + spec); Target target = supervisor.connect(spec); if (! target.isValid()) { - log.log(LogLevel.INFO, "Could not connect"); + log.log(Level.INFO, "Could not connect"); System.exit(1); } else { - log.log(LogLevel.INFO, "Connected to " + spec); + log.log(Level.INFO, "Connected to " + spec); } new RpcTester(target).call(fileReference, filename, blob); @@ -77,12 +77,12 @@ public class RpcTester { void receive(FileReference reference, String filename, byte[] content) { - log.log(LogLevel.INFO, "Preparing receive call for " + reference.value() + " and file " + filename); + log.log(Level.INFO, "Preparing receive call for " + reference.value() + " and file " + filename); XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); Request fileBlob = new Request("filedistribution.receiveFile"); - log.log(LogLevel.INFO, "Calling " + fileBlob.methodName() + " with target " + target); + log.log(Level.INFO, "Calling " + fileBlob.methodName() + " with target " + target); fileBlob.parameters().add(new StringValue(reference.value())); fileBlob.parameters().add(new StringValue(filename)); @@ -90,9 +90,9 @@ public class RpcTester { fileBlob.parameters().add(new Int64Value(hasher.hash(ByteBuffer.wrap(content), 0))); fileBlob.parameters().add(new Int32Value(0)); fileBlob.parameters().add(new StringValue("OK")); - log.log(LogLevel.INFO, "Doing invokeSync"); + log.log(Level.INFO, "Doing invokeSync"); target.invokeSync(fileBlob, 5); - log.log(LogLevel.INFO, "Done with invokeSync"); + log.log(Level.INFO, "Done with invokeSync"); } } } diff --git a/flags/pom.xml b/flags/pom.xml index 87fbc30d92d..ba0e4a94692 100644 --- a/flags/pom.xml +++ b/flags/pom.xml @@ -45,7 +45,7 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> + <artifactId>vespajlib</artifactId> <version>${project.version}</version> <scope>provided</scope> </dependency> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 76206996207..22b5ce419d6 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -150,7 +150,16 @@ public class Flags { "Default number of seconds that a soft start shall use", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - + public static final UnboundDoubleFlag DEFAULT_THREADPOOL_SIZE_FACTOR = defineDoubleFlag( + "default-threadpool-size-factor", 0.0, + "Default multiplication factor when computing maxthreads for main container threadpool based on available cores", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundDoubleFlag DEFAULT_QUEUE_SIZE_FACTOR = defineDoubleFlag( + "default-queue-size-factor", 0.0, + "Default multiplication factor when computing queuesize for burst handling", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag DEFAULT_TOP_K_PROBABILITY = defineDoubleFlag( "default-top-k-probability", 1.0, "Default probability that you will get the globally top K documents when merging many partitions.", @@ -238,12 +247,6 @@ public class Flags { "Takes effect immediately", APPLICATION_ID); - public static final UnboundBooleanFlag NGINX_UPSTREAM_PROXY_PROTOCOL = defineFeatureFlag( - "nginx-upstream-proxy-protocol", false, - "Whether the nginx should enable proxy-protocol for all upstreams", - "Takes effect immediately", - ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag NLB_PROXY_PROTOCOL = defineFeatureFlag( "nlb-proxy-protocol", false, "Configure NLB to use proxy protocol", diff --git a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java index abe8d407ab0..6976654a744 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/file/FlagDbFile.java @@ -1,7 +1,7 @@ // Copyright 2019 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.flags.file; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.json.FlagData; @@ -70,12 +70,12 @@ public class FlagDbFile { FlagData existingFlagData = currentFlagData.get(data.id()); if (existingFlagData == null) { - logger.log(LogLevel.INFO, "New flag " + data.id() + ": " + data.serializeToJson()); + logger.log(Level.INFO, "New flag " + data.id() + ": " + data.serializeToJson()); modified = true; // Could also consider testing with FlagData::equals, but that would be too fragile? } else if (!Objects.equals(data.serializeToJson(), existingFlagData.serializeToJson())){ - logger.log(LogLevel.INFO, "Updating flag " + data.id() + " from " + + logger.log(Level.INFO, "Updating flag " + data.id() + " from " + existingFlagData.serializeToJson() + " to " + data.serializeToJson()); modified = true; } @@ -83,7 +83,7 @@ public class FlagDbFile { if (!flagIdsToBeRemoved.isEmpty()) { String flagIdsString = flagIdsToBeRemoved.stream().map(FlagId::toString).collect(Collectors.joining(", ")); - logger.log(LogLevel.INFO, "Removing flags " + flagIdsString); + logger.log(Level.INFO, "Removing flags " + flagIdsString); modified = true; } diff --git a/fnet/src/examples/frt/rpc/echo_client.cpp b/fnet/src/examples/frt/rpc/echo_client.cpp index cc406224135..06f4ef0ee5b 100644 --- a/fnet/src/examples/frt/rpc/echo_client.cpp +++ b/fnet/src/examples/frt/rpc/echo_client.cpp @@ -84,6 +84,7 @@ EchoClient::Main() printf("Return values != parameters.\n"); } req->SubRef(); + target->SubRef(); return 0; } diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java index 53270f3b51e..18cd6cf02c1 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/athenz/AthenzAuthorizationFilter.java @@ -6,7 +6,7 @@ import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.security.athenz.RequestResourceMapper.ResourceNameAndAction; import com.yahoo.jdisc.http.filter.security.base.JsonSecurityRequestFilterBase; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; @@ -79,7 +79,7 @@ public class AthenzAuthorizationFilter extends JsonSecurityRequestFilterBase { try { Optional<ResourceNameAndAction> resourceMapping = requestResourceMapper.getResourceNameAndAction(request.getMethod(), request.getRequestURI(), request.getQueryString()); - log.log(LogLevel.DEBUG, () -> String.format("Resource mapping for '%s': %s", request, resourceMapping)); + log.log(Level.FINE, () -> String.format("Resource mapping for '%s': %s", request, resourceMapping)); if (resourceMapping.isEmpty()) { incrementAcceptedMetrics(request, false); return Optional.empty(); @@ -92,11 +92,11 @@ public class AthenzAuthorizationFilter extends JsonSecurityRequestFilterBase { incrementAcceptedMetrics(request, true); return Optional.empty(); } - log.log(LogLevel.DEBUG, () -> String.format("Forbidden (403) for '%s': %s", request, resultType.name())); + log.log(Level.FINE, () -> String.format("Forbidden (403) for '%s': %s", request, resultType.name())); incrementRejectedMetrics(request, FORBIDDEN, resultType.name()); return Optional.of(new ErrorResponse(FORBIDDEN, "Access forbidden: " + resultType.getDescription())); } catch (IllegalArgumentException e) { - log.log(LogLevel.DEBUG, () -> String.format("Unauthorized (401) for '%s': %s", request, e.getMessage())); + log.log(Level.FINE, () -> String.format("Unauthorized (401) for '%s': %s", request, e.getMessage())); incrementRejectedMetrics(request, UNAUTHORIZED, "Unauthorized"); return Optional.of(new ErrorResponse(UNAUTHORIZED, e.getMessage())); } @@ -196,7 +196,7 @@ public class AthenzAuthorizationFilter extends JsonSecurityRequestFilterBase { } private static void setAttribute(DiscFilterRequest request, String name, String value) { - log.log(LogLevel.DEBUG, () -> String.format("Setting attribute on '%s': '%s' = '%s'", request, name, value)); + log.log(Level.FINE, () -> String.format("Setting attribute on '%s': '%s' = '%s'", request, name, value)); request.setAttribute(name, value); } diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java index 33d344c4f2a..ec6b1361ebc 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/base/JsonSecurityRequestFilterBase.java @@ -11,7 +11,7 @@ import com.yahoo.jdisc.handler.ResponseDispatch; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.http.filter.DiscFilterRequest; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.UncheckedIOException; import java.util.Optional; @@ -41,7 +41,7 @@ public abstract class JsonSecurityRequestFilterBase extends AbstractComponent im errorMessage.put("message", error.message); error.response.headers().put("Content-Type", "application/json"); // Note: Overwrites header if already exists error.response.headers().put("Cache-Control", "must-revalidate,no-cache,no-store"); - log.log(LogLevel.DEBUG, () -> String.format("Error response for '%s': statusCode=%d, errorCode=%d, message='%s'", + log.log(Level.FINE, () -> String.format("Error response for '%s': statusCode=%d, errorCode=%d, message='%s'", request, error.response.getStatus(), error.errorCode, error.message)); try (FastContentWriter writer = ResponseDispatch.newInstance(error.response).connectFastWriter(responseHandler)) { writer.write(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(errorMessage)); diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java index 4bd78813cba..922e4140868 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusClient.java @@ -9,7 +9,7 @@ import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.RequestDeniedException; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.service.ClientProvider; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; @@ -72,7 +72,7 @@ public final class MbusClient extends AbstractResource implements ClientProvider @Override protected void destroy() { - log.log(LogLevel.DEBUG, "Destroying message bus client."); + log.log(Level.FINE, "Destroying message bus client."); sessionReference.close(); done = true; } @@ -86,7 +86,7 @@ public final class MbusClient extends AbstractResource implements ClientProvider handler.handleResponse(new MbusResponse(StatusCodes.fromMbusReply(reply), reply)) .close(IgnoredCompletionHandler.INSTANCE); } catch (final Exception e) { - log.log(LogLevel.WARNING, "Ignoring exception thrown by ResponseHandler.", e); + log.log(Level.WARNING, "Ignoring exception thrown by ResponseHandler.", e); } } @@ -139,7 +139,7 @@ public final class MbusClient extends AbstractResource implements ClientProvider } sendBlocking(request); } catch (final Exception e) { - log.log(LogLevel.WARNING, "Ignoring exception thrown by MbusClient.", e); + log.log(Level.WARNING, "Ignoring exception thrown by MbusClient.", e); } } } diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java index bc3d1edda7c..e26e1e7e134 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/jdisc/MbusServer.java @@ -10,7 +10,7 @@ import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.handler.ResponseHandler; import com.yahoo.jdisc.service.CurrentContainer; import com.yahoo.jdisc.service.ServerProvider; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.*; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.shared.ServerSession; @@ -42,19 +42,19 @@ public final class MbusServer extends AbstractResource implements ServerProvider @Override public void start() { - log.log(LogLevel.DEBUG, "Starting message bus server."); + log.log(Level.FINE, "Starting message bus server."); running.set(true); } @Override public void close() { - log.log(LogLevel.DEBUG, "Closing message bus server."); + log.log(Level.FINE, "Closing message bus server."); running.set(false); } @Override protected void destroy() { - log.log(LogLevel.DEBUG, "Destroying message bus server."); + log.log(Level.FINE, "Destroying message bus server."); running.set(false); sessionReference.close(); } diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java index fd377aaa7d1..7da164757cd 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedDestinationSession.java @@ -3,7 +3,7 @@ package com.yahoo.messagebus.shared; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.ResourceReference; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.DestinationSession; import com.yahoo.messagebus.DestinationSessionParams; import com.yahoo.messagebus.EmptyReply; @@ -78,7 +78,7 @@ public class SharedDestinationSession extends AbstractResource implements Messag @Override protected void destroy() { - log.log(LogLevel.DEBUG, "Destroying shared destination session."); + log.log(Level.FINE, "Destroying shared destination session."); session.destroy(); mbusReference.close(); } diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java index f543e1b5c8d..5c9fab46e34 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedIntermediateSession.java @@ -3,7 +3,7 @@ package com.yahoo.messagebus.shared; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.ResourceReference; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; import com.yahoo.messagebus.ErrorCode; @@ -97,7 +97,7 @@ public class SharedIntermediateSession extends AbstractResource @Override protected void destroy() { - log.log(LogLevel.DEBUG, "Destroying shared intermediate session."); + log.log(Level.FINE, "Destroying shared intermediate session."); session.destroy(); mbusReference.close(); } diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java index 704a73710cf..dd135a51378 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java @@ -3,7 +3,7 @@ package com.yahoo.messagebus.shared; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.jdisc.AbstractResource; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.DestinationSessionParams; import com.yahoo.messagebus.IntermediateSessionParams; import com.yahoo.messagebus.MessageBus; @@ -35,7 +35,7 @@ public class SharedMessageBus extends AbstractResource { @Override protected void destroy() { - log.log(LogLevel.DEBUG, "Destroying shared message bus."); + log.log(Level.FINE, "Destroying shared message bus."); mbus.destroy(); } diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java index d49a5ea67b8..56071682349 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedSourceSession.java @@ -3,7 +3,7 @@ package com.yahoo.messagebus.shared; import com.yahoo.jdisc.AbstractResource; import com.yahoo.jdisc.ResourceReference; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.ReplyHandler; @@ -50,7 +50,7 @@ public class SharedSourceSession extends AbstractResource implements ClientSessi @Override protected void destroy() { - log.log(LogLevel.DEBUG, "Destroying shared source session."); + log.log(Level.FINE, "Destroying shared source session."); session.close(); mbusReference.close(); } diff --git a/jrt_test/src/tests/echo/echo-client.cpp b/jrt_test/src/tests/echo/echo-client.cpp index c1ffe29e1ce..3a87e38e6e0 100644 --- a/jrt_test/src/tests/echo/echo-client.cpp +++ b/jrt_test/src/tests/echo/echo-client.cpp @@ -77,6 +77,7 @@ public: printf("Return values != parameters.\n"); } req->SubRef(); + target->SubRef(); return 0; } }; diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java index 1c7c71c00b6..0837b25c151 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpLinguistics.java @@ -6,6 +6,8 @@ import com.yahoo.language.detect.Detector; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.simple.SimpleDetector; import com.yahoo.language.simple.SimpleLinguistics; +import java.util.logging.Logger; +import java.util.logging.Level; /** * Returns a linguistics implementation based on OpenNlp, @@ -13,6 +15,7 @@ import com.yahoo.language.simple.SimpleLinguistics; */ public class OpenNlpLinguistics extends SimpleLinguistics { + private static final Logger log = Logger.getLogger(OpenNlpLinguistics.class.getName()); private final Detector detector; public OpenNlpLinguistics() { @@ -26,6 +29,7 @@ public class OpenNlpLinguistics extends SimpleLinguistics { public OpenNlpLinguistics(boolean enableOptimaize) { this(enableOptimaize ? new OptimaizeDetector() : new SimpleDetector()); + log.log(Level.FINE, "using "+(enableOptimaize ? "Optimaize" : "Simple")+" detector"); } private OpenNlpLinguistics(Detector detector) { diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java index 07349811bd4..d3f6fcf2ee3 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java @@ -11,9 +11,12 @@ import opennlp.tools.stemmer.snowball.SnowballStemmer; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.logging.Logger; +import java.util.logging.Level; public class OpenNlpTokenizer implements Tokenizer { private final static int SPACE_CODE = 32; + private static final Logger log = Logger.getLogger(OpenNlpTokenizer.class.getName()); private final Normalizer normalizer; private final Transformer transformer; private final SimpleTokenizer simpleTokenizer; @@ -57,6 +60,7 @@ public class OpenNlpTokenizer implements Tokenizer { } private Stemmer getStemmerForLanguage(Language language, StemMode stemMode) { + log.log(Level.FINEST, () -> "getStemmerForLanguage '"+language+"' mode: "+stemMode); if (language == null || Language.ENGLISH.equals(language) || StemMode.NONE.equals(stemMode)) { return null; } @@ -120,14 +124,21 @@ public class OpenNlpTokenizer implements Tokenizer { private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents, Stemmer stemmer) { + final String original = token; + log.log(Level.FINEST, () -> "processToken '"+original+"'"); token = normalizer.normalize(token); token = LinguisticsCase.toLowerCase(token); if (removeAccents) token = transformer.accentDrop(token, language); if (stemMode != StemMode.NONE) { + final String oldToken = token; token = doStemming(token, stemmer); + final String newToken = token; + log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+newToken+"'"); } - return token; + final String result = token; + log.log(Level.FINEST, () -> "processed token is: "+result); + return result; } private String doStemming(String token, Stemmer stemmer) { diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java index ef1d6c966ac..bf07c91ba44 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OptimaizeDetector.java @@ -22,6 +22,8 @@ import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.util.List; import java.util.Locale; +import java.util.logging.Logger; +import java.util.logging.Level; /** * Detects the language of some sample text using SimpleDetector for CJK and Optimaize otherwise. @@ -33,6 +35,7 @@ public class OptimaizeDetector implements Detector { static private Object initGuard = new Object(); static private TextObjectFactory textObjectFactory = null; static private LanguageDetector languageDetector = null; + static private final Logger log = Logger.getLogger(OptimaizeDetector.class.getName()); static private void initOptimaize() { synchronized (initGuard) { @@ -96,6 +99,7 @@ public class OptimaizeDetector implements Detector { private static Language guessLanguageUsingOptimaize(String input) { Optional<LdLocale> result = languageDetector.detect(textObjectFactory.forText(input)); if ( ! result.isPresent()) return Language.UNKNOWN; + log.log(Level.FINE, () -> "guessing language "+result.get()+" from input: "+input); return Language.fromLocale(new Locale(result.get().getLanguage())); } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index 068fc0126d7..7df432f496d 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -9,6 +9,8 @@ import com.yahoo.language.simple.kstem.KStemmer; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.logging.Logger; +import java.util.logging.Level; /** * <p>A tokenizer which splits on whitespace, normalizes and transforms using the given implementations @@ -25,6 +27,7 @@ public class SimpleTokenizer implements Tokenizer { private final Normalizer normalizer; private final Transformer transformer; private final KStemmer stemmer = new KStemmer(); + private static final Logger log = Logger.getLogger(SimpleTokenizer.class.getName()); public SimpleTokenizer() { this(new SimpleNormalizer(), new SimpleTransformer()); @@ -64,13 +67,21 @@ public class SimpleTokenizer implements Tokenizer { } private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents) { + final String original = token; + log.log(Level.FINEST, () -> "processToken '"+original+"'"); token = normalizer.normalize(token); token = LinguisticsCase.toLowerCase(token); if (removeAccents) token = transformer.accentDrop(token, language); - if (stemMode != StemMode.NONE) + if (stemMode != StemMode.NONE) { + final String oldToken = token; token = stemmer.stem(token); - return token; + final String newToken = token; + log.log(Level.FINEST, () -> "stem '"+oldToken+"' to '"+newToken+"'"); + } + final String result = token; + log.log(Level.FINEST, () -> "processed token is: "+result); + return result; } } diff --git a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java index f2393edeff3..f897933ebe7 100644 --- a/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/AbstractPluginLoader.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.plugin.Plugin; import com.yahoo.plugin.SystemPropertyConfig; @@ -24,7 +24,7 @@ public abstract class AbstractPluginLoader implements PluginLoader { try { plugin = pluginClass.getDeclaredConstructor().newInstance(); } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { - log.log(LogLevel.ERROR, pluginClass.getName() + ": load failed: " + e); + log.log(Level.SEVERE, pluginClass.getName() + ": load failed: " + e); throw new RuntimeException(e); } @@ -34,15 +34,15 @@ public abstract class AbstractPluginLoader implements PluginLoader { String enable = config.get("enable", "true"); if (! enable.equals("true")) { - log.log(LogLevel.INFO, pname + ": plugin disabled by config"); + log.log(Level.INFO, pname + ": plugin disabled by config"); return; } try { plugin.initPlugin(config); - log.log(LogLevel.DEBUG, pname + ": plugin loaded"); + log.log(Level.FINE, pname + ": plugin loaded"); } catch (Exception e) { - log.log(LogLevel.ERROR, pname + ": init failed", e); + log.log(Level.SEVERE, pname + ": init failed", e); } } } diff --git a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java index 32427604bd6..3a87ec920a8 100644 --- a/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java +++ b/logserver/src/main/java/com/yahoo/logserver/BuiltinPluginLoader.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.logserver.handlers.archive.ArchiverPlugin; import com.yahoo.logserver.handlers.logmetrics.LogMetricsPlugin; @@ -17,12 +17,12 @@ public class BuiltinPluginLoader extends AbstractPluginLoader { private static final Logger log = Logger.getLogger(BuiltinPluginLoader.class.getName()); public void loadPlugins() { - log.log(LogLevel.DEBUG, "starting to load builtin plugins"); + log.log(Level.FINE, "starting to load builtin plugins"); loadFromClass(ArchiverPlugin.class); loadFromClass(LogMetricsPlugin.class); - log.log(LogLevel.DEBUG, "done loading builtin plugins"); + log.log(Level.FINE, "done loading builtin plugins"); } } diff --git a/logserver/src/main/java/com/yahoo/logserver/Flusher.java b/logserver/src/main/java/com/yahoo/logserver/Flusher.java index d8d18bd2070..07ecae66d90 100644 --- a/logserver/src/main/java/com/yahoo/logserver/Flusher.java +++ b/logserver/src/main/java/com/yahoo/logserver/Flusher.java @@ -7,7 +7,7 @@ import java.util.Iterator; import java.util.List; import java.util.logging.Logger; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.logserver.handlers.LogHandler; /** @@ -45,13 +45,13 @@ public class Flusher extends Thread { } else { h.flush(); } - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Flushing " + h); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Flushing " + h); } } } } catch (InterruptedException e) { - log.log(LogLevel.WARNING, "flusher was interrupted", e); + log.log(Level.WARNING, "flusher was interrupted", e); } } } diff --git a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java index afc901dc9e2..202d7ac966c 100644 --- a/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java +++ b/logserver/src/main/java/com/yahoo/logserver/LogDispatcher.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogMessage; import com.yahoo.logserver.handlers.LogHandler; @@ -117,8 +117,8 @@ public class LogDispatcher implements LogHandler { } for (LogHandler h : handlers) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Flushing " + h.toString()); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Flushing " + h.toString()); } h.flush(); } @@ -138,7 +138,7 @@ public class LogDispatcher implements LogHandler { } handlers.clear(); - log.log(LogLevel.DEBUG, "Logdispatcher shut down. Handled " + messageCount + " messages"); + log.log(Level.FINE, "Logdispatcher shut down. Handled " + messageCount + " messages"); } /** diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java index 776125fe24b..057460323cd 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/HandlerThread.java @@ -7,7 +7,7 @@ import java.util.List; import java.util.logging.Logger; import com.yahoo.io.FatalErrorHandler; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogMessage; import java.util.concurrent.BlockingQueue; @@ -79,7 +79,7 @@ public class HandlerThread extends Thread implements LogHandler { public HandlerThread(String name) { super(name); queue = new LinkedBlockingQueue<>(queueSize); - log.log(LogLevel.CONFIG, "logserver.queue.size=" + queueSize); + log.log(Level.CONFIG, "logserver.queue.size=" + queueSize); } /** diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java index be788c23e80..ee8c6e36cb8 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/ArchiverHandler.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.logserver.handlers.archive; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogMessage; import com.yahoo.logserver.filter.LogFilter; import com.yahoo.logserver.filter.LogFilterManager; @@ -14,7 +14,6 @@ import java.util.Calendar; import java.util.GregorianCalendar; import java.util.Iterator; import java.util.TimeZone; -import java.util.logging.Level; import java.util.logging.Logger; @@ -212,12 +211,12 @@ public class ArchiverHandler extends AbstractLogHandler { // ensure that root dir exists if (root.isDirectory()) { - log.log(LogLevel.DEBUG, "Using " + absoluteRootDir + " as root"); + log.log(Level.FINE, "Using " + absoluteRootDir + " as root"); } else { if (! root.mkdirs()) { - log.log(LogLevel.ERROR, "Unable to create directory " + absoluteRootDir); + log.log(Level.SEVERE, "Unable to create directory " + absoluteRootDir); } else { - log.log(LogLevel.DEBUG, "Created root at " + absoluteRootDir); + log.log(Level.FINE, "Created root at " + absoluteRootDir); } } filesArchived = new FilesArchived(root); diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java index 83d6a4a0def..9d550a16a7e 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/archive/LogWriter.java @@ -7,7 +7,7 @@ import java.io.IOException; import java.io.Writer; import java.util.logging.Logger; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; /** * This class is not thread-safe. @@ -66,7 +66,7 @@ public class LogWriter { // if file does not exist we have a winner if (! f.exists()) { - log.log(LogLevel.DEBUG, "nextWriter, new file: " + name); + log.log(Level.FINE, "nextWriter, new file: " + name); currentFile = f; bytesWritten = 0; return new FileWriter(f, true); diff --git a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java index fbf38fadc90..9566c534d5d 100644 --- a/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java +++ b/logserver/src/main/java/com/yahoo/logserver/handlers/logmetrics/LogMetricsHandler.java @@ -215,7 +215,7 @@ public class LogMetricsHandler extends AbstractLogHandler { try { Thread.sleep(EVENTINTERVAL * 1000); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, e.getMessage()); + log.log(Level.WARNING, e.getMessage()); } sendEvents(); } diff --git a/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java index 93160c2a7a2..2b702058ede 100644 --- a/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java +++ b/logserver/src/test/java/ai/vespa/logserver/protocol/ArchiveLogMessagesMethodTest.java @@ -10,6 +10,8 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.Transport; import com.yahoo.jrt.Values; +import java.util.logging.Level; + import com.yahoo.log.LogLevel; import com.yahoo.log.LogMessage; import com.yahoo.logserver.LogDispatcher; @@ -33,7 +35,7 @@ public class ArchiveLogMessagesMethodTest { private static final LogMessage MESSAGE_1 = LogMessage.of(Instant.EPOCH.plus(1000, ChronoUnit.DAYS), "localhost", 12, 3456, "my-service", "my-component", LogLevel.ERROR, "My error message"); private static final LogMessage MESSAGE_2 = - LogMessage.of(Instant.EPOCH.plus(5005, ChronoUnit.DAYS), "localhost", 12, 6543, "my-service", "my-component", LogLevel.INFO, "My info message"); + LogMessage.of(Instant.EPOCH.plus(5005, ChronoUnit.DAYS), "localhost", 12, 6543, "my-service", "my-component", Level.INFO, "My info message"); @Test public void server_dispatches_log_messages_from_log_request() { diff --git a/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java b/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java index 76d3bc07d2b..1281bd0b544 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/DynamicThrottlePolicy.java @@ -3,7 +3,7 @@ package com.yahoo.messagebus; import com.yahoo.concurrent.SystemTimer; import com.yahoo.concurrent.Timer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -100,8 +100,8 @@ public class DynamicThrottlePolicy extends StaticThrottlePolicy { } else if (throughput > localMaxThroughput * 1.01) { localMaxThroughput = throughput; windowSize += weight*windowSizeIncrement; - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "windowSize " + windowSize + " throughput " + throughput + " local max " + localMaxThroughput); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "windowSize " + windowSize + " throughput " + throughput + " local max " + localMaxThroughput); } } else { // scale up/down throughput for comparing to window size @@ -119,8 +119,8 @@ public class DynamicThrottlePolicy extends StaticThrottlePolicy { } else { windowSize += weight*windowSizeIncrement; } - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "windowSize " + windowSize + " throughput " + throughput + " local max " + localMaxThroughput + " efficiency " + efficiency); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "windowSize " + windowSize + " throughput " + throughput + " local max " + localMaxThroughput + " efficiency " + efficiency); } } windowSize = Math.max(minWindowSize, windowSize); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index 5622aebab66..feec51643c4 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -3,7 +3,7 @@ package com.yahoo.messagebus; import com.yahoo.concurrent.CopyOnWriteHashMap; import com.yahoo.concurrent.SystemTimer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.NetworkOwner; import com.yahoo.messagebus.routing.Resender; @@ -435,7 +435,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, RoutingTableSpec table = spec.getTable(i); String name = table.getProtocol(); if (!protocolRepository.hasProtocol(name)) { - log.log(LogLevel.INFO, "Protocol '" + name + "' is not supported, ignoring routing table."); + log.log(Level.INFO, "Protocol '" + name + "' is not supported, ignoring routing table."); continue; } tables.put(name, new RoutingTable(table)); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java index 7211e4cead0..0cc8310937b 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.ArrayDeque; import java.util.ArrayList; @@ -168,7 +168,7 @@ public class Messenger implements Runnable { try { task.run(); } catch (final Exception e) { - log.log(LogLevel.ERROR, "An exception was thrown while running " + task.getClass().getName(), e); + log.log(Level.SEVERE, "An exception was thrown while running " + task.getClass().getName(), e); } try { task.destroy(); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/ProtocolRepository.java b/messagebus/src/main/java/com/yahoo/messagebus/ProtocolRepository.java index 102e0be923d..d3ab41331ae 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/ProtocolRepository.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/ProtocolRepository.java @@ -2,7 +2,7 @@ package com.yahoo.messagebus; import com.yahoo.concurrent.CopyOnWriteHashMap; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.routing.RoutingPolicy; import com.yahoo.text.Utf8String; @@ -78,20 +78,20 @@ public class ProtocolRepository { } Protocol protocol = getProtocol(protocolName); if (protocol == null) { - log.log(LogLevel.ERROR, "Protocol '" + protocolName + "' not supported."); + log.log(Level.SEVERE, "Protocol '" + protocolName + "' not supported."); return null; } try { ret = protocol.createPolicy(policyName, policyParam); } catch (RuntimeException e) { - log.log(LogLevel.ERROR, "Protcol '" + protocolName + "' threw an exception: " + e.getMessage(), e); + log.log(Level.SEVERE, "Protcol '" + protocolName + "' threw an exception: " + e.getMessage(), e); if (ret != null) { ret.destroy(); } return null; } if (ret == null) { - log.log(LogLevel.ERROR, "Protocol '" + protocolName + "' failed to create routing policy '" + policyName + + log.log(Level.SEVERE, "Protocol '" + protocolName + "' failed to create routing policy '" + policyName + "' with parameter '" + policyParam + "'."); return null; } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java b/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java index 781235847a0..d955a1a3d34 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/SendProxy.java @@ -4,7 +4,7 @@ package com.yahoo.messagebus; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.routing.Resender; import com.yahoo.messagebus.routing.RoutingNode; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Logger; @@ -39,10 +39,10 @@ public class SendProxy implements MessageHandler, ReplyHandler { public void handleMessage(Message msg) { Trace trace = msg.getTrace(); if (trace.getLevel() == 0) { - if (log.isLoggable(LogLevel.SPAM)) { + if (log.isLoggable(Level.FINEST)) { trace.setLevel(9); logTrace = true; - } else if (log.isLoggable(LogLevel.DEBUG)) { + } else if (log.isLoggable(Level.FINE)) { trace.setLevel(6); logTrace = true; } @@ -59,9 +59,9 @@ public class SendProxy implements MessageHandler, ReplyHandler { Trace trace = msg.getTrace(); if (logTrace) { if (reply.hasErrors()) { - log.log(LogLevel.DEBUG, "Trace for reply with error(s):\n" + reply.getTrace()); - } else if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Trace for reply:\n" + reply.getTrace()); + log.log(Level.FINE, "Trace for reply with error(s):\n" + reply.getTrace()); + } else if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Trace for reply:\n" + reply.getTrace()); } Trace empty = new Trace(); trace.swap(empty); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/TraceNode.java b/messagebus/src/main/java/com/yahoo/messagebus/TraceNode.java index 38963645d54..b6d41a0f112 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/TraceNode.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/TraceNode.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.ArrayList; import java.util.Collections; @@ -440,11 +440,11 @@ public class TraceNode implements Comparable<TraceNode> { node.setStrict(c == '('); } else if (c == ')' || c == '}') { if (node == null) { - log.log(LogLevel.WARNING, "Unexpected closing brace in trace '" + str + "' at position " + i + "."); + log.log(Level.WARNING, "Unexpected closing brace in trace '" + str + "' at position " + i + "."); return new TraceNode(); } if (node.isStrict() != (c == ')')) { - log.log(LogLevel.WARNING, "Mismatched closing brace in trace '" + str + "' at position " + i + "."); + log.log(Level.WARNING, "Mismatched closing brace in trace '" + str + "' at position " + i + "."); return new TraceNode(); } node = node.parent; @@ -452,15 +452,15 @@ public class TraceNode implements Comparable<TraceNode> { } } if (note != null) { - log.log(LogLevel.WARNING, "Unterminated note in trace '" + str + "'."); + log.log(Level.WARNING, "Unterminated note in trace '" + str + "'."); return new TraceNode(); } if (node != proxy) { - log.log(LogLevel.WARNING, "Missing closing brace in trace '" + str + "'."); + log.log(Level.WARNING, "Missing closing brace in trace '" + str + "'."); return new TraceNode(); } if (proxy.getNumChildren() == 0) { - log.log(LogLevel.WARNING, "No nodes found in trace '" + str + "'."); + log.log(Level.WARNING, "No nodes found in trace '" + str + "'."); return new TraceNode(); } if (proxy.getNumChildren() != 1) { diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java index 53c0c855116..2f81e44c74a 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCTarget.java @@ -7,7 +7,7 @@ import com.yahoo.jrt.RequestWaiter; import com.yahoo.jrt.Spec; import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.text.Utf8Array; import java.util.LinkedList; @@ -98,16 +98,16 @@ public class RPCTarget implements RequestWaiter { void resolveVersion(double timeout, VersionHandler handler) { boolean hasVersion = false; boolean shouldInvoke = false; - boolean shouldLog = log.isLoggable(LogLevel.DEBUG); + boolean shouldLog = log.isLoggable(Level.FINE); synchronized (this) { if (version != null) { if (shouldLog) { - log.log(LogLevel.DEBUG, "Version already available for target '" + name + "' (version " + version + ")."); + log.log(Level.FINE, "Version already available for target '" + name + "' (version " + version + ")."); } hasVersion = true; } else { if (shouldLog) { - log.log(LogLevel.DEBUG, "Registering version handler '" + handler + "' for target '" + name + "'."); + log.log(Level.FINE, "Registering version handler '" + handler + "' for target '" + name + "'."); } versionHandlers.add(handler); if (!targetInvoked) { @@ -120,7 +120,7 @@ public class RPCTarget implements RequestWaiter { handler.handleVersion(version); } else if (shouldInvoke) { if (shouldLog) { - log.log(LogLevel.DEBUG, "Invoking mbus.getVersion() on target '" + name + "'"); + log.log(Level.FINE, "Invoking mbus.getVersion() on target '" + name + "'"); } Request req = new Request("mbus.getVersion"); target.invokeAsync(req, timeout, this); @@ -130,7 +130,7 @@ public class RPCTarget implements RequestWaiter { @Override public void handleRequestDone(Request req) { List<VersionHandler> handlers; - boolean shouldLog = log.isLoggable(LogLevel.DEBUG); + boolean shouldLog = log.isLoggable(Level.FINE); synchronized (this) { targetInvoked = false; if (req.checkReturnTypes("s")) { @@ -138,13 +138,13 @@ public class RPCTarget implements RequestWaiter { try { version = new Version(str); if (shouldLog) { - log.log(LogLevel.DEBUG, "Target '" + name + "' has version " + version + "."); + log.log(Level.FINE, "Target '" + name + "' has version " + version + "."); } } catch (IllegalArgumentException e) { - log.log(LogLevel.WARNING, "Failed to parse '" + str + "' as version for target '" + name + "'.", e); + log.log(Level.WARNING, "Failed to parse '" + str + "' as version for target '" + name + "'.", e); } } else { - log.log(LogLevel.INFO, "Method mbus.getVersion() failed for target '" + name + "'; " + + log.log(Level.INFO, "Method mbus.getVersion() failed for target '" + name + "'; " + req.errorMessage()); } handlers = versionHandlers; diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java index b17bb892f7d..1a6944cbae3 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/test/TestServer.java @@ -6,7 +6,7 @@ import com.yahoo.component.Vtag; import com.yahoo.jrt.Spec; import com.yahoo.jrt.slobrok.api.Mirror; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.MessageBus; import com.yahoo.messagebus.MessageBusParams; import com.yahoo.messagebus.Protocol; diff --git a/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java index b5e39035bfb..48b0d355e87 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java @@ -4,7 +4,7 @@ package com.yahoo.messagebus; import com.yahoo.component.Vtag; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.network.rpc.test.TestServer; @@ -64,11 +64,11 @@ public class SendProxyTestCase { LogHandler logHandler = new LogHandler(); log.addHandler(logHandler); - log.setLevel(LogLevel.INFO); + log.setLevel(Level.INFO); sendMessage(0, null); assertNull(logHandler.trace); - log.setLevel(LogLevel.DEBUG); + log.setLevel(Level.FINE); sendMessage(0, null); assertNull(logHandler.trace); @@ -90,7 +90,7 @@ public class SendProxyTestCase { "</trace>\n", logHandler.trace); logHandler.trace = null; - log.setLevel(LogLevel.SPAM); + log.setLevel(Level.FINEST); sendMessage(1, null); assertNull(logHandler.trace); diff --git a/metrics-proxy/pom.xml b/metrics-proxy/pom.xml index 355f420c2a4..8bf5a30e584 100644 --- a/metrics-proxy/pom.xml +++ b/metrics-proxy/pom.xml @@ -97,12 +97,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>yolean</artifactId> <version>${project.version}</version> <scope>provided</scope> diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index 53a05ef88f0..33827634ebf 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -23,7 +23,7 @@ import java.util.stream.Collectors; import static ai.vespa.metricsproxy.metric.ExternalMetrics.extractConfigserverDimensions; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.stream.Collectors.toList; /** @@ -91,17 +91,17 @@ public class MetricsManager { public List<MetricsPacket.Builder> getMetricsAsBuilders(List<VespaService> services, Instant startTime) { if (services.isEmpty()) return Collections.emptyList(); - log.log(DEBUG, () -> "Updating services prior to fetching metrics, number of services= " + services.size()); + log.log(FINE, () -> "Updating services prior to fetching metrics, number of services= " + services.size()); vespaServices.updateServices(services); List<MetricsPacket.Builder> result = vespaMetrics.getMetrics(services); - log.log(DEBUG, () -> "Got " + result.size() + " metrics packets for vespa services."); + log.log(FINE, () -> "Got " + result.size() + " metrics packets for vespa services."); purgeStaleMetrics(); List<MetricsPacket.Builder> externalPackets = externalMetrics.getMetrics().stream() .filter(MetricsPacket.Builder::hasMetrics) .collect(toList()); - log.log(DEBUG, () -> "Got " + externalPackets.size() + " external metrics packets with whitelisted metrics."); + log.log(FINE, () -> "Got " + externalPackets.size() + " external metrics packets with whitelisted metrics."); result.addAll(externalPackets); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java index d20d4b0cb3c..4d57e89fc76 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/ApplicationMetricsRetriever.java @@ -6,7 +6,7 @@ import ai.vespa.metricsproxy.metric.model.MetricsPacket; import ai.vespa.util.http.VespaHttpClientBuilder; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.apache.http.client.HttpClient; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; @@ -69,7 +69,7 @@ public class ApplicationMetricsRetriever extends AbstractComponent { } public Map<Node, List<MetricsPacket.Builder>> getMetrics(ConsumerId consumer) { - log.log(LogLevel.DEBUG, () -> "Retrieving metrics from " + clients.size() + " nodes."); + log.log(Level.FINE, () -> "Retrieving metrics from " + clients.size() + " nodes."); var forkJoinTask = forkJoinPool.submit(() -> clients.parallelStream() .map(client -> getNodeMetrics(client, consumer)) .collect(toMap(Map.Entry::getKey, Map.Entry::getValue))); @@ -77,7 +77,7 @@ public class ApplicationMetricsRetriever extends AbstractComponent { try { var metricsByNode = forkJoinTask.get(taskTimeout.toMillis(), TimeUnit.MILLISECONDS); - log.log(LogLevel.DEBUG, () -> "Finished retrieving metrics from " + clients.size() + " nodes."); + log.log(Level.FINE, () -> "Finished retrieving metrics from " + clients.size() + " nodes."); return metricsByNode; } catch (Exception e) { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java index 145eef3f745..a93b5eb31c0 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/NodeMetricsClient.java @@ -20,7 +20,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Logger; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.Collections.emptyList; /** @@ -65,13 +65,13 @@ public class NodeMetricsClient { private Snapshot retrieveMetrics(ConsumerId consumer) { String metricsUri = node.metricsUri(consumer).toString(); - log.log(DEBUG, () -> "Retrieving metrics from host " + metricsUri); + log.log(FINE, () -> "Retrieving metrics from host " + metricsUri); try { String metricsJson = httpClient.execute(new HttpGet(metricsUri), new BasicResponseHandler()); var newMetrics = GenericJsonUtil.toMetricsPackets(metricsJson); snapshotsRetrieved ++; - log.log(DEBUG, () -> "Successfully retrieved " + newMetrics.size() + " metrics packets from " + metricsUri); + log.log(FINE, () -> "Successfully retrieved " + newMetrics.size() + " metrics packets from " + metricsUri); return new Snapshot(Instant.now(clock), newMetrics); } catch (IOException e) { log.warning("Unable to retrieve metrics from " + metricsUri + ": " + Exceptions.toMessageString(e)); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java index 0a2dbc9a910..337967f3075 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/metric/ExternalMetrics.java @@ -21,7 +21,7 @@ import java.util.logging.Logger; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.metric.model.ServiceId.toServiceId; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.stream.Collectors.toCollection; /** @@ -53,7 +53,7 @@ public class ExternalMetrics { public void setExtraMetrics(List<MetricsPacket.Builder> externalPackets) { // TODO: Metrics filtering per consumer is not yet implemented. // Split each packet per metric, and re-aggregate based on the metrics each consumer wants. Then filter out all packages with no consumers. - log.log(DEBUG, () -> "Setting new external metrics with " + externalPackets.size() + " metrics packets."); + log.log(FINE, () -> "Setting new external metrics with " + externalPackets.size() + " metrics packets."); externalPackets.forEach(packet -> { packet.addConsumers(consumers.getAllConsumers()) .retainMetrics(metricsToRetain()) diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java index 269b70c8316..c09dc44d3fd 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcConnector.java @@ -11,7 +11,7 @@ import com.yahoo.jrt.Transport; import java.util.logging.Logger; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; /** @@ -30,7 +30,7 @@ public class RpcConnector extends AbstractComponent { Spec spec = new Spec(config.port()); try { acceptor = supervisor.listen(spec); - log.log(DEBUG, "Listening on " + spec.host() + ":" + acceptor.port()); + log.log(FINE, "Listening on " + spec.host() + ":" + acceptor.port()); } catch (ListenFailedException e) { stop(); log.log(INFO, "Failed listening at " + spec.host() + ":" + spec.port()); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java index ba84347c7ea..30e7d7b81f8 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/rpc/RpcServer.java @@ -20,7 +20,7 @@ import static ai.vespa.metricsproxy.metric.model.ConsumerId.toConsumerId; import static ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil.toMetricsPackets; import static ai.vespa.metricsproxy.metric.model.json.YamasJsonUtil.toYamasArray; import static com.yahoo.collections.CollectionUtil.mkString; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; import static java.util.logging.Level.WARNING; @@ -46,7 +46,7 @@ public class RpcServer { this.vespaServices = vespaServices; this.metricsManager = metricsManager; addMethods(connector); - log.log(DEBUG, "RPC server created"); + log.log(FINE, "RPC server created"); } private void addMethods(RpcConnector connector) { @@ -120,13 +120,13 @@ public class RpcServer { Instant startTime = Instant.now(); req.detach(); String service = req.parameters().get(0).asString(); - log.log(DEBUG, () -> "getMetricsForYamas called at " + startTime + " with argument: " + service); + log.log(FINE, () -> "getMetricsForYamas called at " + startTime + " with argument: " + service); List<VespaService> services = vespaServices.getMonitoringServices(service); - log.log(DEBUG, () -> "Getting metrics for services: " + mkString(services, "[", ", ", "]")); + log.log(FINE, () -> "Getting metrics for services: " + mkString(services, "[", ", ", "]")); if (services.isEmpty()) setNoServiceError(req, service); else withExceptionHandling(req, () -> { List<MetricsPacket> packets = metricsManager.getMetrics(services, startTime); - log.log(DEBUG,() -> "Returning metrics packets:\n" + mkString(packets, "\n")); + log.log(FINE,() -> "Returning metrics packets:\n" + mkString(packets, "\n")); req.returnValues().add(new StringValue(toYamasArray(packets).serialize())); }); req.returnRequest(); @@ -146,7 +146,7 @@ public class RpcServer { void setExtraMetrics(Request req) { String metricsJson = req.parameters().get(0).asString(); - log.log(DEBUG, "setExtraMetrics called with argument: " + metricsJson); + log.log(FINE, "setExtraMetrics called with argument: " + metricsJson); withExceptionHandling(req, () -> metricsManager.setExtraMetrics(toMetricsPackets(metricsJson))); } @@ -197,9 +197,9 @@ public class RpcServer { } private void logSpentTime() { - Level logLevel = DEBUG; + Level logLevel = Level.FINE; if (spentTime() > LOG_SPENT_TIME_LIMIT) { - logLevel = INFO; + logLevel = Level.INFO; } if (log.isLoggable(logLevel)) { log.log(logLevel, "RPC request '" + request.methodName() + "' with parameters '" + diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java index b5bd6c073cd..c0e3e636aa6 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/ConfigSentinelClient.java @@ -3,7 +3,7 @@ package ai.vespa.metricsproxy.service; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.jrt.ErrorCode; import com.yahoo.jrt.Request; @@ -45,7 +45,7 @@ public class ConfigSentinelClient extends AbstractComponent { try { setStatus(services); } catch (Exception e) { - log.log(LogLevel.ERROR, "Unable to update service pids from sentinel", e); + log.log(Level.SEVERE, "Unable to update service pids from sentinel", e); } } @@ -58,11 +58,11 @@ public class ConfigSentinelClient extends AbstractComponent { synchronized void ping(VespaService s) { List<VespaService> services = new ArrayList<>(); services.add(s); - log.log(LogLevel.DEBUG, "Ping for service " + s); + log.log(Level.FINE, "Ping for service " + s); try { setStatus(services); } catch (Exception e) { - log.log(LogLevel.ERROR, "Unable to update service pids from sentinel", e); + log.log(Level.SEVERE, "Unable to update service pids from sentinel", e); } } @@ -92,7 +92,7 @@ public class ConfigSentinelClient extends AbstractComponent { //from the sentinel for (VespaService s : services) { if ((!s.getServiceName().equals("configserver")) && !updatedServices.contains(s)) { - log.log(LogLevel.DEBUG,"Service " + s + " is no longer found with sentinel - setting alive = false"); + log.log(Level.FINE,"Service " + s + " is no longer found with sentinel - setting alive = false"); s.setAlive(false); } } @@ -166,10 +166,10 @@ public class ConfigSentinelClient extends AbstractComponent { { servicelist = req.returnValues().get(0).asString(); } else { - log.log(LogLevel.WARNING, "Bad answer to RPC request: " + req.errorMessage()); + log.log(Level.WARNING, "Bad answer to RPC request: " + req.errorMessage()); } } else { - log.log(LogLevel.WARNING, "Could not connect to sentinel at: "+spec); + log.log(Level.WARNING, "Could not connect to sentinel at: "+spec); } return servicelist; } finally { diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java index 9068be81b65..d1e1d6f14cc 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/HttpMetricFetcher.java @@ -2,7 +2,7 @@ package ai.vespa.metricsproxy.service; import ai.vespa.util.http.VespaHttpClientBuilder; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.yolean.Exceptions; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.HttpGet; @@ -40,11 +40,11 @@ public abstract class HttpMetricFetcher { String u = "http://localhost:" + port + path; this.url = URI.create(u); - log.log(LogLevel.DEBUG, "Fetching metrics from " + u + " with timeout " + CONNECTION_TIMEOUT); + log.log(Level.FINE, "Fetching metrics from " + u + " with timeout " + CONNECTION_TIMEOUT); } String getJson() throws IOException { - log.log(LogLevel.DEBUG, "Connecting to url " + url + " for service '" + service + "'"); + log.log(Level.FINE, "Connecting to url " + url + " for service '" + service + "'"); return httpClient.execute(new HttpGet(url), new BasicResponseHandler()); } @@ -64,17 +64,17 @@ public abstract class HttpMetricFetcher { private void logMessage(String message, int timesFetched) { if (service.isAlive() && timesFetched > 5) { - log.log(LogLevel.INFO, message); + log.log(Level.INFO, message); } else { - log.log(LogLevel.DEBUG, message); + log.log(Level.FINE, message); } } void logMessageNoResponse(String message, int timesFetched) { if (timesFetched > 5) { - log.log(LogLevel.WARNING, message); + log.log(Level.WARNING, message); } else { - log.log(LogLevel.INFO, message); + log.log(Level.INFO, message); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java index ec36ae344bc..1cab1a859a9 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/RemoteHealthMetricFetcher.java @@ -2,7 +2,7 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.HealthMetric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.json.JSONException; import org.json.JSONObject; @@ -64,7 +64,7 @@ public class RemoteHealthMetricFetcher extends HttpMetricFetcher { return HealthMetric.get(code, message); } catch (JSONException e) { - log.log(LogLevel.DEBUG, "Failed to parse json response from metrics page:" + e + ":" + data); + log.log(Level.FINE, "Failed to parse json response from metrics page:" + e + ":" + data); return HealthMetric.getUnknown("Not able to parse json from status page"); } } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java index c8fbc83eb59..a6c2220b5a2 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/SystemPoller.java @@ -3,7 +3,7 @@ package ai.vespa.metricsproxy.service; import ai.vespa.metricsproxy.metric.Metric; import ai.vespa.metricsproxy.metric.Metrics; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.BufferedReader; import java.io.FileNotFoundException; @@ -81,7 +81,7 @@ public class SystemPoller { br.close(); } catch (IOException ex) { - log.log(LogLevel.DEBUG, "Unable to read line from smaps file", ex); + log.log(Level.FINE, "Unable to read line from smaps file", ex); return size; } @@ -101,7 +101,7 @@ public class SystemPoller { return; } - log.log(LogLevel.DEBUG, "Monitoring system metrics for " + services.size() + " services"); + log.log(Level.FINE, "Monitoring system metrics for " + services.size() + " services"); long sysJiffies = getNormalizedSystemJiffies(); for (VespaService s : services) { @@ -112,10 +112,10 @@ public class SystemPoller { } Metrics metrics = new Metrics(); - log.log(LogLevel.DEBUG, "Current size of system metrics for service " + s + " is " + metrics.size()); + log.log(Level.FINE, "Current size of system metrics for service " + s + " is " + metrics.size()); long[] size = getMemoryUsage(s); - log.log(LogLevel.DEBUG, "Updating memory metric for service " + s); + log.log(Level.FINE, "Updating memory metric for service " + s); metrics.add(new Metric("memory_virt", size[memoryTypeVirtual], startTime / 1000)); metrics.add(new Metric("memory_rss", size[memoryTypeResident], startTime / 1000)); @@ -152,7 +152,7 @@ public class SystemPoller { try { in = new BufferedReader(new FileReader("/proc/" + pid + "/stat")); } catch (FileNotFoundException ex) { - log.log(LogLevel.DEBUG, "Unable to find pid " + pid + " in proc directory, for service " + service.getInstanceName()); + log.log(Level.FINE, "Unable to find pid " + pid + " in proc directory, for service " + service.getInstanceName()); service.setAlive(false); return 0; } @@ -161,7 +161,7 @@ public class SystemPoller { line = in.readLine(); in.close(); } catch (IOException ex) { - log.log(LogLevel.DEBUG, "Unable to read line from process stat file", ex); + log.log(Level.FINE, "Unable to read line from process stat file", ex); return 0; } @@ -180,7 +180,7 @@ public class SystemPoller { try { in = new BufferedReader(new FileReader("/proc/stat")); } catch (FileNotFoundException ex) { - log.log(LogLevel.ERROR, "Unable to open stat file", ex); + log.log(Level.SEVERE, "Unable to open stat file", ex); return 0; } try { @@ -194,7 +194,7 @@ public class SystemPoller { in.close(); } catch (IOException ex) { - log.log(LogLevel.ERROR, "Unable to read line from stat file", ex); + log.log(Level.SEVERE, "Unable to read line from stat file", ex); return 0; } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java index 76a4ae5046b..e01a68a1f7f 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/service/VespaServices.java @@ -15,7 +15,7 @@ import java.util.logging.Logger; import static ai.vespa.metricsproxy.core.MetricsConsumers.toUnmodifiableLinkedMap; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; -import static com.yahoo.log.LogLevel.DEBUG; +import static java.util.logging.Level.FINE; /** * Creates representations for the Vespa services running on the node, @@ -47,12 +47,12 @@ public class VespaServices { private List<VespaService> createServices(VespaServicesConfig servicesConfig, String monitoringSystemName) { List<VespaService> services = new ArrayList<>(); for (Service s : servicesConfig.service()) { - log.log(DEBUG, "Creating service " + s.name()); + log.log(FINE, "Creating service " + s.name()); VespaService vespaService = VespaService.create(s.name(), s.configId(), s.port(), monitoringSystemName, createServiceDimensions(s)); services.add(vespaService); } - log.log(DEBUG, "Created new services: " + services.size()); + log.log(FINE, "Created new services: " + services.size()); return services; } @@ -62,7 +62,7 @@ public class VespaServices { */ public final void updateServices(List<VespaService> services) { if (sentinel != null) { - log.log(DEBUG, "Updating services "); + log.log(FINE, "Updating services "); sentinel.updateServiceStatuses(services); } } @@ -103,7 +103,7 @@ public class VespaServices { List<VespaService> myServices = new ArrayList<>(); for (VespaService s : services) { - log.log(DEBUG, () -> "getMonitoringServices. service=" + service + ", checking against " + s + ", which has monitoring name " + s.getMonitoringName()); + log.log(FINE, () -> "getMonitoringServices. service=" + service + ", checking against " + s + ", which has monitoring name " + s.getMonitoringName()); if (s.getMonitoringName().equalsIgnoreCase(service)) { myServices.add(s); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java index 9600dfca042..5c7d8b3ce83 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/Telegraf.java @@ -3,7 +3,7 @@ package ai.vespa.metricsproxy.telegraf; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.system.execution.ProcessExecutor; import com.yahoo.system.execution.ProcessResult; import org.apache.velocity.VelocityContext; @@ -76,7 +76,7 @@ public class Telegraf extends AbstractComponent { ProcessResult processResult = uncheck(() -> processExecutor.execute(command)) .orElseThrow(() -> new RuntimeException("Timed out running command: " + command)); - logger.log(LogLevel.DEBUG, () -> String.format("Exit code: %d\nstdOut: %s\nstdErr: %s", + logger.log(Level.FINE, () -> String.format("Exit code: %d\nstdOut: %s\nstdErr: %s", processResult.exitCode, processResult.stdOut, processResult.stdErr)); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java index 23da51ea082..3864d25c2a6 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/telegraf/TelegrafRegistry.java @@ -1,7 +1,7 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.metricsproxy.telegraf; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.ArrayList; import java.util.Collections; @@ -18,12 +18,12 @@ public class TelegrafRegistry { private static final Logger logger = Logger.getLogger(TelegrafRegistry.class.getName()); public void addInstance(Telegraf telegraf) { - logger.log(LogLevel.DEBUG, () -> "Adding Telegraf instance to registry: " + telegraf.hashCode()); + logger.log(Level.FINE, () -> "Adding Telegraf instance to registry: " + telegraf.hashCode()); telegrafInstances.add(telegraf); } public void removeInstance(Telegraf telegraf) { - logger.log(LogLevel.DEBUG, () -> "Removing Telegraf instance from registry: " + telegraf.hashCode()); + logger.log(Level.FINE, () -> "Removing Telegraf instance from registry: " + telegraf.hashCode()); telegrafInstances.remove(telegraf); } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java index 2b0af93fd8e..d326291a5a5 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/MatMul.java @@ -126,7 +126,7 @@ public class MatMul extends IntermediateOperation { // a1 < a2 < a3 < a4 for (int j = i+1; j < typeA.rank(); ++j) { String jDim = typeA.dimensionNames().get(j); - renamer.addConstraint(iDim, jDim, DimensionRenamer.Constraint.lessThan(true), this); + renamer.addConstraint(iDim, jDim, DimensionRenamer.Constraint.lessThan(false), this); } // not equal to last 2 dimensions in B for (int j = typeB.rank()-2; j < typeB.rank(); ++j) { @@ -148,7 +148,7 @@ public class MatMul extends IntermediateOperation { // b1 < b2 < b3 < b4 for (int j = i+1; j < typeB.rank(); ++j) { String jDim = typeB.dimensionNames().get(j); - renamer.addConstraint(iDim, jDim, DimensionRenamer.Constraint.lessThan(true), this); + renamer.addConstraint(iDim, jDim, DimensionRenamer.Constraint.lessThan(false), this); } // not equal to last 2 dimensions in A for (int j = typeA.rank()-2; j < typeA.rank(); ++j) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java index b22c76a987f..b84e25f59ed 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/TaskContext.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Level; import java.util.logging.Logger; @@ -28,7 +28,7 @@ public interface TaskContext { } /** - * Log message at LogLevel.INFO, scoped to denote the current task. The message may + * Log message at Level.INFO, scoped to denote the current task. The message may * also be directed to status pages or similar. * * Please do not call this too many times as that spams the log. Typically a task may call @@ -37,7 +37,7 @@ public interface TaskContext { * Do not log a message that is also recorded with recordSystemModification. */ default void log(Logger logger, String message) { - log(logger, LogLevel.INFO, message); + log(logger, Level.INFO, message); } default void log(Logger logger, String messageFormat, Object... args) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java index 22633f67463..93541b03537 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeMembership.java @@ -1,25 +1,27 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; +import java.util.Objects; + /** * @author freva */ public class NodeMembership { - private final String clusterType; + private final ClusterType clusterType; private final String clusterId; private final String group; private final int index; private final boolean retired; public NodeMembership(String clusterType, String clusterId, String group, int index, boolean retired) { - this.clusterType = clusterType; + this.clusterType = new ClusterType(clusterType); this.clusterId = clusterId; this.group = group; this.index = index; this.retired = retired; } - public String clusterType() { + public ClusterType type() { return clusterType; } @@ -74,4 +76,39 @@ public class NodeMembership { " retired = " + retired + " }"; } + + public static class ClusterType { + private final String type; + + private ClusterType(String type) { + this.type = Objects.requireNonNull(type); + } + + public boolean isAdmin() { return "admin".equals(type); } + public boolean isContent() { return "content".equals(type) || "combined".equals(type); } + public boolean isContainer() { return "container".equals(type) || "combined".equals(type); } + + public String value() { + return type; + } + + @Override + public String toString() { + return type; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ClusterType that = (ClusterType) o; + return type.equals(that.type); + } + + @Override + public int hashCode() { + return type.hashCode(); + } + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java index 85c35369a22..76616a3a8f5 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeSpec.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; import java.time.Instant; import java.util.EnumSet; @@ -220,6 +221,10 @@ public class NodeSpec { return resources.memoryGb(); } + public DiskSize diskSize() { + return DiskSize.of(resources.diskGb(), DiskSize.Unit.GB); + } + public double diskGb() { return resources.diskGb(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java index f5bb46ef62a..d7f248250a8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/docker/DockerOperationsImpl.java @@ -88,7 +88,7 @@ public class DockerOperationsImpl implements DockerOperations { if (context.nodeType() != NodeType.proxy && context.nodeType() != NodeType.controller) command.withSecurityOpt("no-new-privileges"); - if (context.node().membership().map(NodeMembership::clusterType).map("content"::equalsIgnoreCase).orElse(false)) + if (context.node().membership().map(m -> m.type().isContent()).orElse(false)) command.withSecurityOpt("seccomp=unconfined"); DockerNetworking networking = context.dockerNetworking(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 1684c0dd2a1..ef515022ae6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -3,12 +3,20 @@ package com.yahoo.vespa.hosted.node.admin.maintenance; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.yahoo.log.LogLevel; +import com.yahoo.config.provision.NodeType; +import java.util.logging.Level; import com.yahoo.vespa.hosted.dockerapi.Container; +import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; +import com.yahoo.vespa.hosted.node.admin.maintenance.disk.CoredumpCleanupRule; +import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup; +import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule; +import com.yahoo.vespa.hosted.node.admin.maintenance.disk.LinearCleanupRule; +import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; +import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; @@ -17,18 +25,20 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Clock; import java.time.Duration; +import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.logging.Logger; -import java.util.regex.Pattern; -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameMatches; -import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.olderThan; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; import static com.yahoo.yolean.Exceptions.uncheck; /** @@ -42,44 +52,43 @@ public class StorageMaintainer { private final Terminal terminal; private final CoredumpHandler coredumpHandler; private final Path archiveContainerStoragePath; + private final DiskCleanup diskCleanup; private final Clock clock; // We cache disk usage to avoid doing expensive disk operations so often - private final Cache<Path, Long> diskUsage = CacheBuilder.newBuilder() + private final Cache<ContainerName, DiskSize> diskUsage = CacheBuilder.newBuilder() .maximumSize(100) .expireAfterWrite(5, TimeUnit.MINUTES) .build(); public StorageMaintainer(Terminal terminal, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath) { - this(terminal, coredumpHandler, archiveContainerStoragePath, Clock.systemUTC()); + this(terminal, coredumpHandler, archiveContainerStoragePath, new DiskCleanup(), Clock.systemUTC()); } - public StorageMaintainer(Terminal terminal, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath, Clock clock) { + public StorageMaintainer(Terminal terminal, CoredumpHandler coredumpHandler, Path archiveContainerStoragePath, DiskCleanup diskCleanup, Clock clock) { this.terminal = terminal; this.coredumpHandler = coredumpHandler; this.archiveContainerStoragePath = archiveContainerStoragePath; + this.diskCleanup = diskCleanup; this.clock = clock; } - public Optional<Long> getDiskUsageFor(NodeAgentContext context) { + public Optional<DiskSize> diskUsageFor(NodeAgentContext context) { try { - Path path = context.pathOnHostFromPathInNode("/"); - - Long cachedDiskUsage = diskUsage.getIfPresent(path); + DiskSize cachedDiskUsage = diskUsage.getIfPresent(context.containerName()); if (cachedDiskUsage != null) return Optional.of(cachedDiskUsage); - long diskUsageBytes = getDiskUsedInBytes(context, path); - diskUsage.put(path, diskUsageBytes); + DiskSize diskUsageBytes = getDiskUsed(context, context.pathOnHostFromPathInNode("/")); + diskUsage.put(context.containerName(), diskUsageBytes); return Optional.of(diskUsageBytes); } catch (Exception e) { - context.log(logger, LogLevel.WARNING, "Failed to get disk usage", e); + context.log(logger, Level.WARNING, "Failed to get disk usage", e); return Optional.empty(); } } - // Public for testing - long getDiskUsedInBytes(TaskContext context, Path path) { - if (!Files.exists(path)) return 0; + DiskSize getDiskUsed(TaskContext context, Path path) { + if (!Files.exists(path)) return DiskSize.ZERO; String output = terminal.newCommandLine(context) .add("du", "-xsk", path.toString()) @@ -89,43 +98,50 @@ public class StorageMaintainer { String[] results = output.split("\t"); if (results.length != 2) { - throw new RuntimeException("Result from disk usage command not as expected: " + output); + throw new ConvergenceException("Result from disk usage command not as expected: " + output); } - return 1024 * Long.parseLong(results[0]); + return DiskSize.of(Long.parseLong(results[0]), DiskSize.Unit.kiB); } + public boolean cleanDiskIfFull(NodeAgentContext context) { + double totalBytes = context.node().diskSize().bytes(); + // Delete enough bytes to get below 80% disk usage, but only if we are already using more than 90% disk + long bytesToRemove = diskUsageFor(context) + .map(diskUsage -> (long) (diskUsage.bytes() - 0.8 * totalBytes)) + .filter(bytes -> bytes > totalBytes * 0.1) + .orElse(0L); + + if (bytesToRemove > 0 && diskCleanup.cleanup(context, createCleanupRules(context), bytesToRemove)) { + diskUsage.invalidate(context.containerName()); + return true; + } + return false; + } - /** Deletes old log files for vespa and nginx */ - public void removeOldFilesFromNode(NodeAgentContext context) { - Path[] logPaths = { - context.pathInNodeUnderVespaHome("logs/nginx"), - context.pathInNodeUnderVespaHome("logs/vespa") - }; + private List<DiskCleanupRule> createCleanupRules(NodeAgentContext context) { + Instant start = clock.instant(); + double oneMonthSeconds = Duration.ofDays(30).getSeconds(); + Function<Instant, Double> monthNormalizer = instant -> Duration.between(instant, start).getSeconds() / oneMonthSeconds; + Function<String, Path> pathOnHostUnderContainerVespaHome = path -> + context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome(path)); + List<DiskCleanupRule> rules = new ArrayList<>(); - for (Path pathToClean : logPaths) { - Path path = context.pathOnHostFromPathInNode(pathToClean); - FileFinder.files(path) - .match(olderThan(Duration.ofDays(3)).and(nameMatches(Pattern.compile(".*\\.log.+")))) - .maxDepth(1) - .deleteRecursively(context); - } + rules.add(CoredumpCleanupRule.forContainer(pathOnHostUnderContainerVespaHome.apply("var/crash"))); - FileFinder.files(context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("logs/vespa/qrs"))) - .match(olderThan(Duration.ofDays(3))) - .deleteRecursively(context); + if (context.node().membership().map(m -> m.type().isContainer()).orElse(false)) + rules.add(new LinearCleanupRule(() -> FileFinder.files(pathOnHostUnderContainerVespaHome.apply("logs/vespa/qrs")).list(), + fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.HIGHEST)); - FileFinder.files(context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("logs/vespa/logarchive"))) - .match(olderThan(Duration.ofDays(31))) - .deleteRecursively(context); + if (context.nodeType() == NodeType.tenant && context.node().membership().map(m -> m.type().isAdmin()).orElse(false)) + rules.add(new LinearCleanupRule(() -> FileFinder.files(pathOnHostUnderContainerVespaHome.apply("logs/vespa/logarchive")).list(), + fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.HIGHEST)); - FileFinder.directories(context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("var/db/vespa/filedistribution"))) - .match(olderThan(Duration.ofDays(31))) - .deleteRecursively(context); + if (context.nodeType() == NodeType.proxy) + rules.add(new LinearCleanupRule(() -> FileFinder.files(pathOnHostUnderContainerVespaHome.apply("logs/nginx")).list(), + fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.MEDIUM)); - FileFinder.directories(context.pathOnHostFromPathInNode(context.pathInNodeUnderVespaHome("var/db/vespa/download"))) - .match(olderThan(Duration.ofDays(31))) - .deleteRecursively(context); + return rules; } /** Checks if container has any new coredumps, reports and archives them if so */ @@ -174,11 +190,11 @@ public class StorageMaintainer { String output = uncheck(() -> Files.readAllLines(Paths.get("/proc/cpuinfo")).stream() .filter(line -> line.startsWith("microcode")) .findFirst() - .orElseThrow(() -> new RuntimeException("No microcode information found in /proc/cpuinfo"))); + .orElseThrow(() -> new ConvergenceException("No microcode information found in /proc/cpuinfo"))); String[] results = output.split(":"); if (results.length != 2) { - throw new RuntimeException("Result from detect microcode command not as expected: " + output); + throw new ConvergenceException("Result from detect microcode command not as expected: " + output); } return results[1].trim(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java index 19abe784749..360cea8a60d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.acl; import com.google.common.net.InetAddresses; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.node.admin.docker.DockerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.Editor; @@ -99,14 +99,14 @@ public class AclMaintainer { dockerOperations.executeCommandInNetworkNamespace(context, ipVersion.iptablesRestore(), fileHandler.absolutePath()); } catch (Exception e) { if (flush) { - context.log(logger, LogLevel.ERROR, "Exception occurred while syncing iptable " + table + ", attempting rollback", e); + context.log(logger, Level.SEVERE, "Exception occurred while syncing iptable " + table + ", attempting rollback", e); try { dockerOperations.executeCommandInNetworkNamespace(context, ipVersion.iptablesCmd(), "-F", "-t", table); } catch (Exception ne) { - context.log(logger, LogLevel.ERROR, "Rollback of table " + table + " failed, giving up", ne); + context.log(logger, Level.SEVERE, "Rollback of table " + table + " failed, giving up", ne); } } else { - context.log(logger, LogLevel.WARNING, "Unable to sync iptables for " + table, e); + context.log(logger, Level.WARNING, "Unable to sync iptables for " + table, e); } } }; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 6aa5a8d9fc1..c6c024ca658 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -2,10 +2,8 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions; import com.yahoo.vespa.hosted.dockerapi.metrics.Metrics; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembership; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; @@ -44,7 +42,7 @@ public class CoredumpHandler { private static final String LZ4_PATH = "/usr/bin/lz4"; private static final String PROCESSING_DIRECTORY_NAME = "processing"; private static final String METADATA_FILE_NAME = "metadata.json"; - private static final String COREDUMP_FILENAME_PREFIX = "dump_"; + public static final String COREDUMP_FILENAME_PREFIX = "dump_"; private final Logger logger = Logger.getLogger(CoredumpHandler.class.getName()); private final ObjectMapper objectMapper = new ObjectMapper(); @@ -166,6 +164,7 @@ public class CoredumpHandler { Path coredumpFilePathInContainer = context.pathInNodeFromPathOnHost(coredumpFilePathOnHost); Map<String, Object> metadata = coreCollector.collect(context, coredumpFilePathInContainer); metadata.putAll(nodeAttributesSupplier.get()); + metadata.put("coredump_path", doneCoredumpsPath.resolve(context.containerName().asString()).resolve(coredumpDirectory.getFileName()).toString()); String metadataFields = objectMapper.writeValueAsString(Map.of("fields", metadata)); metadataPath.writeUtf8File(metadataFields); @@ -245,7 +244,7 @@ public class CoredumpHandler { node.membership().ifPresent(membership -> dimensionsBuilder - .add("clustertype", membership.clusterType()) + .add("clustertype", membership.type().value()) .add("clusterid", membership.clusterId()) ); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java new file mode 100644 index 00000000000..2ad887514e5 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRule.java @@ -0,0 +1,106 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; + +import java.nio.file.Path; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.temporal.ChronoField; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.nameStartsWith; + +/** + * @author freva + */ +public class CoredumpCleanupRule { + + private static final Comparator<FileAttributes> CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR = Comparator + .comparing((FileAttributes fa) -> !fa.filename().contains("vespa-")) + .thenComparing(FileAttributes::lastModifiedTime); + + public static DiskCleanupRule forContainer(Path containerCrashPath) { + return new ContainerCoredumpCleanupRule(containerCrashPath); + } + + public static DiskCleanupRule forHost(Path processedCoredumpsPath) { + return new HostCoredumpCleanupRule(processedCoredumpsPath); + } + + /** Assigns MEDIUM priority to the oldest, unprocessed coredump and HIGHEST for the remaining */ + private static class ContainerCoredumpCleanupRule implements DiskCleanupRule { + private final Path containerCrashPath; + + private ContainerCoredumpCleanupRule(Path containerCrashPath) { + this.containerCrashPath = containerCrashPath; + } + + @Override + public Collection<PrioritizedFileAttributes> prioritize() { + List<FileAttributes> fileAttributes = FileFinder.files(containerCrashPath) + .maxDepth(1).stream() + .sorted(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR) + .collect(Collectors.toList()); + + return mapFirstAndRemaining(fileAttributes, Priority.MEDIUM, Priority.HIGHEST).collect(Collectors.toList()); + } + } + + /** Assigns MEDIUM priority to the first coredump of the day for each container, HIGH for the remaining */ + private static class HostCoredumpCleanupRule implements DiskCleanupRule { + private final Path processedCoredumpsPath; + + private HostCoredumpCleanupRule(Path processedCoredumpsPath) { + this.processedCoredumpsPath = processedCoredumpsPath; + } + + @Override + public Collection<PrioritizedFileAttributes> prioritize() { + Map<String, List<FileAttributes>> fileAttributesByContainerDay = FileFinder.files(processedCoredumpsPath) + .match(nameStartsWith(CoredumpHandler.COREDUMP_FILENAME_PREFIX)) + .stream() + .sorted(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR) + .collect(Collectors.groupingBy( + // Group FileAttributes by string [container-name]_[day of year], e.g. zt00534-v6-2_234 + fa -> containerNameFromProcessedCoredumpPath(fa.path()) + "_" + dayOfYear(fa.lastModifiedTime()), + Collectors.collectingAndThen( + Collectors.toCollection(ArrayList::new), + l -> { l.sort(CORE_DUMP_FILE_ATTRIBUTE_COMPARATOR); return l; } ))); + + return fileAttributesByContainerDay.values().stream() + .flatMap(fa -> mapFirstAndRemaining(fa, Priority.MEDIUM, Priority.HIGH)) + .collect(Collectors.toList()); + } + } + + /** + * Maps list of FileAttributes into list of PrioritizedFileAttributes where the first FileAttribute is given + * {@code first} priority, while the remaining FileAttributes are given {@code remaining} priority */ + private static Stream<PrioritizedFileAttributes> mapFirstAndRemaining(List<FileAttributes> fileAttributes, Priority first, Priority remaining) { + return IntStream.range(0, fileAttributes.size()) + .mapToObj(i -> new PrioritizedFileAttributes(fileAttributes.get(i), i == 0 ? first : remaining)); + } + + /** Extracts container-name from path under processed-coredumps or empty string */ + private static String containerNameFromProcessedCoredumpPath(Path path) { + if (path.getNameCount() < 3) return ""; // Path is too short + return path.getName(path.getNameCount() - 3).toString(); + } + + /** Returns day number of the year (1-365 (or 366 for leap years)) */ + private static int dayOfYear(Instant instant) { + return instant.atOffset(ZoneOffset.UTC).get(ChronoField.DAY_OF_YEAR); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java new file mode 100644 index 00000000000..0ce6f1e6b45 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanup.java @@ -0,0 +1,59 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.logging.Logger; + +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; +import static com.yahoo.yolean.Exceptions.uncheck; + +/** + * @author freva + */ +public class DiskCleanup { + + private static final Logger logger = Logger.getLogger(DiskCleanup.class.getName()); + private static final Comparator<PrioritizedFileAttributes> PRIORITIZED_FILE_ATTRIBUTES_COMPARATOR = Comparator + .comparing(PrioritizedFileAttributes::priority) + .thenComparingLong(f -> f.fileAttributes().size()) + .reversed(); + + public boolean cleanup(TaskContext context, List<DiskCleanupRule> rules, long bytesToRemove) { + if (bytesToRemove <= 0) return false; + + long[] btr = new long[] { bytesToRemove }; + List<Path> deletedPaths = new ArrayList<>(); + try { + rules.stream() + .flatMap(rule -> rule.prioritize().stream()) + .sorted(PRIORITIZED_FILE_ATTRIBUTES_COMPARATOR) + .takeWhile(fa -> btr[0] > 0) + .forEach(pfa -> { + if (uncheck(() -> Files.deleteIfExists(pfa.fileAttributes().path()))) { + btr[0] -= pfa.fileAttributes().size(); + deletedPaths.add(pfa.fileAttributes().path()); + } + }); + + } finally { + String wantedDeleteSize = DiskSize.of(bytesToRemove).asString(); + String deletedSize = DiskSize.of(bytesToRemove - btr[0]).asString(); + if (deletedPaths.size() > 20) { + context.log(logger, "Deleted %d files (%s) because disk was getting full", deletedPaths.size(), deletedSize); + } else if (deletedPaths.size() > 0) { + context.log(logger, "Deleted %s because disk was getting full from: %s", deletedSize, deletedPaths); + } else { + context.log(logger, "Wanted to delete %s, but failed to find any files to delete", wantedDeleteSize); + } + } + + return !deletedPaths.isEmpty(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java new file mode 100644 index 00000000000..a4c8229d0d7 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupRule.java @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; + +import java.util.Collection; + +/** + * @author freva + */ +public interface DiskCleanupRule { + + Collection<PrioritizedFileAttributes> prioritize(); + + enum Priority { + LOWEST, LOW, MEDIUM, HIGH, HIGHEST + } + + class PrioritizedFileAttributes { + private final FileFinder.FileAttributes fileAttributes; + private final Priority priority; + + public PrioritizedFileAttributes(FileFinder.FileAttributes fileAttributes, Priority priority) { + this.fileAttributes = fileAttributes; + this.priority = priority; + } + + public Priority priority() { return priority; } + public FileFinder.FileAttributes fileAttributes() { return fileAttributes; } + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java new file mode 100644 index 00000000000..3ef4e2e4f6a --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRule.java @@ -0,0 +1,48 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; + +/** + * Prioritizes files by first scoring them with the given scoring function and then mapping the scores to a + * priority within the given range. + * The priority room is evenly split between given lowest and highest priority for range [0, 1.0). Scores below 0 + * are assigned lowest, while scores at or higher than 1 are assigned highest priority. + * + * Typical use-case is for log files. The scoring function calculates the file age and normalizes it by dividing it + * by expected max age of log files. The oldest logs will then by prioritized by highest given priority. + * + * @author freva + */ +public class LinearCleanupRule implements DiskCleanupRule { + private final Supplier<List<FileAttributes>> lister; + private final Function<FileAttributes, Priority> prioritizer; + + public LinearCleanupRule(Supplier<List<FileAttributes>> lister, + Function<FileAttributes, Double> scorer, Priority lowest, Priority highest) { + if (lowest.ordinal() > highest.ordinal()) + throw new IllegalArgumentException("Lowest priority: " + lowest + " is higher than highest priority: " + highest); + + this.lister = lister; + + Priority[] values = Priority.values(); + int range = highest.ordinal() - lowest.ordinal() + 1; + this.prioritizer = fa -> { + int ordinal = (int) (lowest.ordinal() + scorer.apply(fa) * range); + return values[Math.max(lowest.ordinal(), Math.min(highest.ordinal(), ordinal))]; + }; + } + + @Override + public Collection<PrioritizedFileAttributes> prioritize() { + return lister.get().stream() + .map(fa -> new PrioritizedFileAttributes(fa, prioritizer.apply(fa))) + .collect(Collectors.toList()); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java new file mode 100644 index 00000000000..dff40015522 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/package-info.java @@ -0,0 +1,5 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 4c5ad7c5c3a..3320851a36c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance.identity; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyUtils; @@ -109,7 +109,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { public boolean converge(NodeAgentContext context) { try { - context.log(logger, LogLevel.DEBUG, "Checking certificate"); + context.log(logger, Level.FINE, "Checking certificate"); Path containerSiaDirectory = context.pathOnHostFromPathInNode(CONTAINER_SIA_DIRECTORY); Path privateKeyFile = SiaUtils.getPrivateKeyFile(containerSiaDirectory, context.identity()); Path certificateFile = SiaUtils.getCertificateFile(containerSiaDirectory, context.identity()); @@ -136,7 +136,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { if (shouldRefreshCredentials(age)) { context.log(logger, "Certificate is ready to be refreshed (age=%s)", age.toString()); if (shouldThrottleRefreshAttempts(context.containerName(), now)) { - context.log(logger, LogLevel.WARNING, String.format( + context.log(logger, Level.WARNING, String.format( "Skipping refresh attempt as last refresh was on %s (less than %s ago)", lastRefreshAttempt.get(context.containerName()).toString(), REFRESH_BACKOFF.toString())); return false; @@ -146,7 +146,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { return true; } } - context.log(logger, LogLevel.DEBUG, "Certificate is still valid"); + context.log(logger, Level.FINE, "Certificate is still valid"); return false; } catch (IOException e) { throw new UncheckedIOException(e); @@ -169,7 +169,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { Instant expiry = certificate.getNotAfter().toInstant(); return Duration.between(now, expiry); } catch (IOException e) { - context.log(logger, LogLevel.ERROR, "Unable to read certificate at " + certificateFile, e); + context.log(logger, Level.SEVERE, "Unable to read certificate at " + certificateFile, e); return Duration.ZERO; } } @@ -236,14 +236,14 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { context.log(logger, "Instance successfully refreshed and credentials written to file"); } catch (ZtsClientException e) { if (e.getErrorCode() == 403 && e.getDescription().startsWith("Certificate revoked")) { - context.log(logger, LogLevel.ERROR, "Certificate cannot be refreshed as it is revoked by ZTS - re-registering the instance now", e); + context.log(logger, Level.SEVERE, "Certificate cannot be refreshed as it is revoked by ZTS - re-registering the instance now", e); registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile); } else { throw e; } } } catch (Exception e) { - context.log(logger, LogLevel.ERROR, "Certificate refresh failed: " + e.getMessage(), e); + context.log(logger, Level.SEVERE, "Certificate refresh failed: " + e.getMessage(), e); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java index e6d6f9463d3..453f4f4b4c1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminStateUpdater.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeadmin; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.config.provision.HostName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; @@ -180,9 +180,9 @@ public class NodeAdminStateUpdater { .collect(Collectors.toSet()); nodeAdmin.refreshContainersToRun(nodeAgentContexts); } catch (ConvergenceException e) { - log.log(LogLevel.WARNING, "Failed to update which containers should be running: " + Exceptions.toMessageString(e)); + log.log(Level.WARNING, "Failed to update which containers should be running: " + Exceptions.toMessageString(e)); } catch (RuntimeException e) { - log.log(LogLevel.WARNING, "Failed to update which containers should be running", e); + log.log(Level.WARNING, "Failed to update which containers should be running", e); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 9c8b0ec2b86..eb3bd0d6d2e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -5,7 +5,7 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.flags.DoubleFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; @@ -48,9 +48,6 @@ import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.Containe */ public class NodeAgentImpl implements NodeAgent { - // This is used as a definition of 1 GB when comparing flavor specs in node-repo - private static final long BYTES_IN_GB = 1_000_000_000L; - // Container is started with uncapped CPU and is kept that way until the first successful health check + this duration // Subtract 1 second to avoid warmup coming in lockstep with tick time and always end up using an extra tick when there are just a few ms left private static final Duration DEFAULT_WARM_UP_DURATION = Duration.ofSeconds(90).minus(Duration.ofSeconds(1)); @@ -167,7 +164,7 @@ public class NodeAgentImpl implements NodeAgent { void resumeNodeIfNeeded(NodeAgentContext context) { if (!hasResumedNode) { - context.log(logger, LogLevel.DEBUG, "Starting optional node program resume command"); + context.log(logger, Level.FINE, "Starting optional node program resume command"); dockerOperations.resumeNode(context); hasResumedNode = true; } @@ -293,7 +290,7 @@ public class NodeAgentImpl implements NodeAgent { } catch (RuntimeException e) { // It's bad to continue as-if nothing happened, but on the other hand if we do not proceed to // remove container, we will not be able to upgrade to fix any problems in the suspend logic! - context.log(logger, LogLevel.WARNING, "Failed trying to suspend container", e); + context.log(logger, Level.WARNING, "Failed trying to suspend container", e); } } @@ -343,7 +340,7 @@ public class NodeAgentImpl implements NodeAgent { } stopServices(context); } catch (Exception e) { - context.log(logger, LogLevel.WARNING, "Failed stopping services, ignoring", e); + context.log(logger, Level.WARNING, "Failed stopping services, ignoring", e); } } @@ -399,18 +396,18 @@ public class NodeAgentImpl implements NodeAgent { public void converge(NodeAgentContext context) { try { doConverge(context); - context.log(logger, LogLevel.INFO, "Converged"); + context.log(logger, Level.INFO, "Converged"); } catch (ConvergenceException e) { context.log(logger, e.getMessage()); } catch (ContainerNotFoundException e) { containerState = ABSENT; - context.log(logger, LogLevel.WARNING, "Container unexpectedly gone, resetting containerState to " + containerState); + context.log(logger, Level.WARNING, "Container unexpectedly gone, resetting containerState to " + containerState); } catch (DockerException e) { numberOfUnhandledException++; - context.log(logger, LogLevel.ERROR, "Caught a DockerException", e); + context.log(logger, Level.SEVERE, "Caught a DockerException", e); } catch (Throwable e) { numberOfUnhandledException++; - context.log(logger, LogLevel.ERROR, "Unhandled exception, ignoring", e); + context.log(logger, Level.SEVERE, "Unhandled exception, ignoring", e); } } @@ -445,13 +442,9 @@ public class NodeAgentImpl implements NodeAgent { stopServicesIfNeeded(context); break; case active: + storageMaintainer.cleanDiskIfFull(context); storageMaintainer.handleCoreDumpsForContainer(context, container); - storageMaintainer.getDiskUsageFor(context) - .map(diskUsage -> (double) diskUsage / BYTES_IN_GB / node.diskGb()) - .filter(diskUtil -> diskUtil >= 0.8) - .ifPresent(diskUtil -> storageMaintainer.removeOldFilesFromNode(context)); - if (downloadImageIfNeeded(node, container)) { context.log(logger, "Waiting for image to download " + context.node().wantedDockerImage().get().asString()); return; @@ -517,7 +510,7 @@ public class NodeAgentImpl implements NodeAgent { StringBuilder builder = new StringBuilder(); appendIfDifferent(builder, "state", lastNode, node, NodeSpec::state); if (builder.length() > 0) { - context.log(logger, LogLevel.INFO, "Changes to node: " + builder.toString()); + context.log(logger, Level.INFO, "Changes to node: " + builder.toString()); } } @@ -575,7 +568,7 @@ public class NodeAgentImpl implements NodeAgent { try { aclMaintainer.ifPresent(maintainer -> maintainer.converge(context)); } catch (RuntimeException suppressed) { - logger.log(LogLevel.WARNING, "Suppressing ACL update failure: " + suppressed); + logger.log(Level.WARNING, "Suppressing ACL update failure: " + suppressed); e.addSuppressed(suppressed); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java new file mode 100644 index 00000000000..40a11f61d3f --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSize.java @@ -0,0 +1,65 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import java.util.Locale; +import java.util.Objects; + +/** + * @author freva + */ +public class DiskSize { + + public static final DiskSize ZERO = DiskSize.of(0); + private static final char[] UNITS = "kMGTPE".toCharArray(); + + public enum Unit { + kB(1000), kiB(1 << 10), + MB(1_000_000), MiB(1 << 20), + GB(1_000_000_000), GiB(1 << 30), + PB(1_000_000_000_000L), PiB(1L << 40); + + private final long size; + Unit(long size) { this.size = size; } + } + + private final long bytes; + private DiskSize(long bytes) { this.bytes = bytes; } + + public long bytes() { return bytes; } + public long as(Unit unit) { return bytes / unit.size; } + public double asDouble(Unit unit) { return (double) bytes / unit.size; } + + public DiskSize add(DiskSize other) { return new DiskSize(bytes + other.bytes); } + + public static DiskSize of(long bytes) { return new DiskSize(bytes); } + public static DiskSize of(double bytes, Unit unit) { return new DiskSize((long) (bytes * unit.size)); } + public static DiskSize of(long bytes, Unit unit) { return new DiskSize(bytes * unit.size); } + + public String asString() { return asString(0); } + public String asString(int decimals) { + if (bytes < 1000) return bytes + " bytes"; + + int unit = -1; + double remaining = bytes; + for (; remaining >= 1000; unit++) remaining /= 1000; + return String.format(Locale.ENGLISH, "%." + decimals + "f %sB", remaining, UNITS[unit]); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DiskSize size = (DiskSize) o; + return bytes == size.bytes; + } + + @Override + public int hashCode() { + return Objects.hash(bytes); + } + + @Override + public String toString() { + return asString(); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java index f10ebd5e315..f9a369aab27 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinder.java @@ -14,6 +14,8 @@ import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Set; @@ -35,6 +37,7 @@ public class FileFinder { private static final Logger logger = Logger.getLogger(FileFinder.class.getName()); private final Path basePath; + private final Set<Path> pruned = new HashSet<>(); private Predicate<FileAttributes> matcher; private int maxDepth = Integer.MAX_VALUE; @@ -63,7 +66,7 @@ public class FileFinder { /** * Predicate that will be used to match files and directories under the base path. * - * NOTE: Consequtive calls to this method are ANDed (this include the initial filter from + * NOTE: Consecutive calls to this method are ANDed (this include the initial filter from * {@link #files(Path)} or {@link #directories(Path)}. */ public FileFinder match(Predicate<FileAttributes> matcher) { @@ -72,6 +75,27 @@ public class FileFinder { } /** + * Path for which whole directory tree will be skipped, including the path itself. + * The path must be under {@code basePath} or be relative to {@code basePath}. + */ + public FileFinder prune(Path path) { + if (!path.isAbsolute()) + path = basePath.resolve(path); + + if (!path.startsWith(basePath)) + throw new IllegalArgumentException("Prune path " + path + " is not under base path " + basePath); + + this.pruned.add(path); + return this; + } + + /** Convenience method for pruning multiple paths, see {@link #prune(Path)}. */ + public FileFinder prune(Collection<Path> paths) { + paths.forEach(this::prune); + return this; + } + + /** * Maximum depth (relative to basePath) where contents should be matched with the given filters. * Default is unlimited. */ @@ -145,6 +169,8 @@ public class FileFinder { @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { + if (pruned.contains(dir)) return FileVisitResult.SKIP_SUBTREE; + currentLevel++; FileAttributes attributes = new FileAttributes(dir, attrs); @@ -193,7 +219,7 @@ public class FileFinder { private final Path path; private final BasicFileAttributes attributes; - FileAttributes(Path path, BasicFileAttributes attributes) { + public FileAttributes(Path path, BasicFileAttributes attributes) { this.path = path; this.attributes = attributes; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java index 0c03dc7f483..668bffd570a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ChildProcess2Impl.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.process; import com.yahoo.jdisc.Timer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.nio.file.Files; import java.nio.file.Path; @@ -92,7 +92,7 @@ public class ChildProcess2Impl implements ChildProcess2 { if (commandLine.getOutputFile().isEmpty()) Files.delete(outputPath); } catch (Throwable t) { - logger.log(LogLevel.WARNING, "Failed to delete " + outputPath, t); + logger.log(Level.WARNING, "Failed to delete " + outputPath, t); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java index dbe6b984211..f4be8b7479f 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessFactoryImpl.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.process; import com.yahoo.jdisc.Timer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.io.File; import java.io.IOException; @@ -88,7 +88,7 @@ public class ProcessFactoryImpl implements ProcessFactory { if (commandLine.getOutputFile().isEmpty()) Files.delete(outputFile); } catch (IOException ioException) { - logger.log(LogLevel.WARNING, "Failed to delete temporary file at " + + logger.log(Level.WARNING, "Failed to delete temporary file at " + outputFile, ioException); } throw throwable; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java index 3d2caaa2ce9..4fb7ff5571a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/process/ProcessStarterImpl.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.task.util.process; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.util.logging.Logger; @@ -16,8 +16,8 @@ public class ProcessStarterImpl implements ProcessStarter { @Override public ProcessApi2 start(ProcessBuilder processBuilder) { - if (logger.isLoggable(LogLevel.DEBUG)) { - logger.log(LogLevel.DEBUG, "Spawning process: " + processBuilder.command()); + if (logger.isLoggable(Level.FINE)) { + logger.log(Level.FINE, "Spawning process: " + processBuilder.command()); } Process process = uncheck(processBuilder::start); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 4e0fd95384c..2e40596bcf0 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; +import com.yahoo.vespa.hosted.provision.restapi.NodesV2ApiHandler; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import org.junit.After; import org.junit.Before; @@ -52,7 +53,7 @@ public class RealNodeRepositoryTest { * Starts NodeRepository with * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeFlavors} * {@link com.yahoo.vespa.hosted.provision.testutils.MockNodeRepository} - * {@link com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler} + * {@link NodesV2ApiHandler} * These classes define some test data that is used in these tests. */ @Before diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java index 9b509829ba2..a1ea3558504 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integrationTests/DockerTester.java @@ -66,7 +66,7 @@ public class DockerTester implements AutoCloseable { DockerTester() { - when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.empty()); + when(storageMaintainer.diskUsageFor(any())).thenReturn(Optional.empty()); IPAddressesMock ipAddresses = new IPAddressesMock(); ipAddresses.addAddress(HOST_HOSTNAME.value(), "1.1.1.1"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index 50a8da63b96..c17d0017269 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -1,10 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.maintenance; +import com.yahoo.config.provision.NodeResources; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; +import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; +import com.yahoo.vespa.hosted.node.admin.task.util.file.DiskSize; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.After; @@ -25,6 +29,11 @@ import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; /** * @author dybis @@ -44,17 +53,17 @@ public class StorageMaintainerTest { Files.createDirectories(context.pathOnHostFromPathInNode("/")); terminal.expectCommand("du -xsk /home/docker/host-1 2>&1", 0, "321\t/home/docker/host-1/"); - assertEquals(Optional.of(328_704L), storageMaintainer.getDiskUsageFor(context)); + assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); // Value should still be cached, no new execution against the terminal - assertEquals(Optional.of(328_704L), storageMaintainer.getDiskUsageFor(context)); + assertEquals(Optional.of(DiskSize.of(328_704)), storageMaintainer.diskUsageFor(context)); } @Test public void testNonExistingDiskUsed() { StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, null, null); - long usedBytes = storageMaintainer.getDiskUsedInBytes(null, Paths.get("/fake/path")); - assertEquals(0L, usedBytes); + DiskSize size = storageMaintainer.getDiskUsed(null, Paths.get("/fake/path")); + assertEquals(DiskSize.ZERO, size); } @After @@ -85,7 +94,7 @@ public class StorageMaintainerTest { // Archive container-1 ManualClock clock = new ManualClock(Instant.ofEpochSecond(1234567890)); - StorageMaintainer storageMaintainer = new StorageMaintainer(null, null, pathToArchiveDir, clock); + StorageMaintainer storageMaintainer = new StorageMaintainer(null, null, pathToArchiveDir, null, clock); storageMaintainer.archiveNodeStorage(context1); clock.advance(Duration.ofSeconds(3)); @@ -140,4 +149,39 @@ public class StorageMaintainerTest { return context; } } + + public static class CleanupTests { + + private final TestTerminal terminal = new TestTerminal(); + private final DiskCleanup diskCleanup = mock(DiskCleanup.class); + private final ManualClock clock = new ManualClock(); + private final StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, null, null, diskCleanup, clock); + private final FileSystem fileSystem = TestFileSystem.create(); + private final NodeAgentContext context = new NodeAgentContextImpl + .Builder(NodeSpec.Builder.testSpec("h123a.domain.tld").resources(new NodeResources(1, 1, 1, 1)).build()) + .pathToContainerStorageFromFileSystem(fileSystem).build(); + + @Test + public void not_run_if_not_enough_used() throws IOException { + Files.createDirectories(context.pathOnHostFromPathInNode("/")); + mockDiskUsage(500L); + + storageMaintainer.cleanDiskIfFull(context); + verifyNoInteractions(diskCleanup); + } + + @Test + public void deletes_correct_amount() throws IOException { + Files.createDirectories(context.pathOnHostFromPathInNode("/")); + mockDiskUsage(950_000L); + + storageMaintainer.cleanDiskIfFull(context); + // Allocated size: 1 GB, usage: 950_000 kiB (972.8 MB). Wanted usage: 80% => 800 MB + verify(diskCleanup).cleanup(eq(context), any(), eq(172_800_000L)); + } + + private void mockDiskUsage(long kBytes) { + terminal.expectCommand("du -xsk /home/docker/h123a 2>&1", 0, kBytes + "\t/path"); + } + } } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index 60a4462e9e9..8aa4fb9131b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -159,6 +159,7 @@ public class CoredumpHandlerTest { "\"backtrace\":[\"call 1\",\"function 2\",\"something something\"]," + "\"vespa_version\":\"6.48.4\"," + "\"bin_path\":\"/bin/bash\"," + + "\"coredump_path\":\"/home/docker/dumps/container-123/id-123\"," + "\"docker_image\":\"vespa/ci:6.48.4\"" + "}}"; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java new file mode 100644 index 00000000000..e8ca2ff8491 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/CoredumpCleanupRuleTest.java @@ -0,0 +1,103 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +public class CoredumpCleanupRuleTest { + + private final FileSystem fileSystem = TestFileSystem.create(); + + @Test + public void for_container_test() throws IOException { + Path path = fileSystem.getPath("/test/path"); + DiskCleanupRule rule = CoredumpCleanupRule.forContainer(path); + + assertPriorities(rule, Map.of()); + + createFile(path.resolve("core1"), Instant.ofEpochSecond(232)); + assertPriorities(rule, Map.of("/test/path/core1", Priority.MEDIUM)); + + createFile(path.resolve("core2"), Instant.ofEpochSecond(123)); + assertPriorities(rule, Map.of( + "/test/path/core2", Priority.MEDIUM, + "/test/path/core1", Priority.HIGHEST)); + + createFile(path.resolve("vespa-proton-bin.core.325"), Instant.ofEpochSecond(456)); + createFile(path.resolve("vespa-distributor.core.764"), Instant.ofEpochSecond(256)); + var expected = Map.of( + "/test/path/core2", Priority.HIGHEST, + "/test/path/core1", Priority.HIGHEST, + "/test/path/vespa-proton-bin.core.325", Priority.HIGHEST, + "/test/path/vespa-distributor.core.764", Priority.MEDIUM); + assertPriorities(rule, expected); + + // processing core has no effect on this + Files.createDirectories(path.resolve("processing/abcd-1234")); + createFile(path.resolve("processing/abcd-1234/core5"), Instant.ofEpochSecond(67)); + assertPriorities(rule, expected); + } + + @Test + public void for_host_test() throws IOException { + Path path = fileSystem.getPath("/test/path"); + DiskCleanupRule rule = CoredumpCleanupRule.forHost(path); + + assertPriorities(rule, Map.of()); + + createFile(path.resolve("h123a/abcd-1234/dump_core1"), Instant.parse("2020-04-21T19:21:00Z")); + createFile(path.resolve("h123a/abcd-1234/metadata.json"), Instant.parse("2020-04-21T19:26:00Z")); + assertPriorities(rule, Map.of("/test/path/h123a/abcd-1234/dump_core1", Priority.MEDIUM)); + + createFile(path.resolve("h123a/abcd-efgh/dump_core1"), Instant.parse("2020-04-21T07:13:00Z")); + createFile(path.resolve("h123a/56ad-af42/dump_vespa-distributor.321"), Instant.parse("2020-04-21T23:37:00Z")); + createFile(path.resolve("h123a/4324-a23d/dump_core2"), Instant.parse("2020-04-22T04:56:00Z")); + createFile(path.resolve("h123a/8534-7da3/dump_vespa-proton-bin.123"), Instant.parse("2020-04-19T15:35:00Z")); + + // Also create a core for a second container: h123b + createFile(path.resolve("h123b/db1a-ab34/dump_core1"), Instant.parse("2020-04-21T07:01:00Z")); + createFile(path.resolve("h123b/7392-59ad/dump_vespa-proton-bin.342"), Instant.parse("2020-04-22T12:05:00Z")); + + assertPriorities(rule, Map.of( + "/test/path/h123a/abcd-1234/dump_core1", Priority.HIGH, + "/test/path/h123a/abcd-efgh/dump_core1", Priority.HIGH, + + // Although it is the oldest core of the day for h123a, it is the first one that starts with vespa- + "/test/path/h123a/56ad-af42/dump_vespa-distributor.321", Priority.MEDIUM, + "/test/path/h123a/4324-a23d/dump_core2", Priority.MEDIUM, + "/test/path/h123a/8534-7da3/dump_vespa-proton-bin.123", Priority.MEDIUM, + "/test/path/h123b/db1a-ab34/dump_core1", Priority.MEDIUM, + "/test/path/h123b/7392-59ad/dump_vespa-proton-bin.342", Priority.MEDIUM + )); + } + + private static void createFile(Path path, Instant instant) throws IOException { + Files.createDirectories(path.getParent()); + Files.createFile(path); + Files.setLastModifiedTime(path, FileTime.from(instant)); + } + + private static void assertPriorities(DiskCleanupRule rule, Map<String, Priority> expected) { + Map<String, Priority> actual = rule.prioritize().stream() + .collect(Collectors.toMap(pfa -> pfa.fileAttributes().path().toString(), PrioritizedFileAttributes::priority)); + + assertEquals(new TreeMap<>(expected), new TreeMap<>(actual)); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java new file mode 100644 index 00000000000..7065b261b1b --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/DiskCleanupTest.java @@ -0,0 +1,129 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFileAttributes; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author freva + */ +public class DiskCleanupTest { + + private final TestTaskContext context = new TestTaskContext(); + private final DiskCleanupTester tester = new DiskCleanupTester(); + private final DiskCleanup diskCleanup = new DiskCleanup(); + + @Test + public void nothing_deleted() throws IOException { + assertFalse(diskCleanup.cleanup(context, List.of(), 0)); + assertFalse(diskCleanup.cleanup(context, List.of(), 10)); + + DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock(); + DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock(); + assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 0)); + assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + + tester.createFile("/path/that-should-not-be-deleted", 5); + assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + tester.assertAllFilesExistExcept(); + + // Create a file and let rule return it, but before cleanup is run, the file is deleted + rule1.addFile(tester.createFile("/path/file-does-not-exist", 1), Priority.HIGHEST); + Files.delete(tester.path("/path/file-does-not-exist")); + assertFalse(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + } + + @Test + public void delete_test() throws IOException { + tester.createFile("/opt/vespa/var/db/do-not-delete-1.db", 1); + tester.createFile("/opt/vespa/var/db/do-not-delete-2.db", 1); + tester.createFile("/opt/vespa/var/zookeeper/do-not-delete-3", 1); + tester.createFile("/opt/vespa/var/index/something-important", 1); + + DiskCleanupRuleMock rule1 = new DiskCleanupRuleMock() + .addFile(tester.createFile("/opt/vespa/logs/vespa-1.log", 10), Priority.MEDIUM) + .addFile(tester.createFile("/opt/vespa/logs/vespa-2.log", 8), Priority.HIGH) + .addFile(tester.createFile("/opt/vespa/logs/vespa-3.log", 13), Priority.HIGHEST) + .addFile(tester.createFile("/opt/vespa/logs/vespa-4.log", 10), Priority.HIGHEST); + DiskCleanupRuleMock rule2 = new DiskCleanupRuleMock() + .addFile(tester.createFile("/opt/vespa/var/crash/core1", 105), Priority.LOW) + .addFile(tester.createFile("/opt/vespa/var/crash/vespa-proton-bin.core-232", 190), Priority.HIGH) + .addFile(tester.createFile("/opt/vespa/var/crash/core3", 54), Priority.MEDIUM) + .addFile(tester.createFile("/opt/vespa/var/crash/core4", 300), Priority.HIGH); + + // 2 files with HIGHEST priority, tie broken by the largest size which is won by "vespa-3.log", since + // it is >= 10 bytes, no more files are deleted + assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log"); + + // Called with the same arguments, but vespa-3.log is still missing... + assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 10)); + tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log"); + + assertTrue(diskCleanup.cleanup(context, List.of(rule1, rule2), 500)); + tester.assertAllFilesExistExcept("/opt/vespa/logs/vespa-3.log", "/opt/vespa/logs/vespa-4.log", // from before + // 300 + 190 + 8 + 54 + "/opt/vespa/var/crash/core4", "/opt/vespa/var/crash/vespa-proton-bin.core-232", "/opt/vespa/logs/vespa-2.log", "/opt/vespa/var/crash/core3"); + } + + private static class DiskCleanupRuleMock implements DiskCleanupRule { + private final ArrayList<PrioritizedFileAttributes> pfa = new ArrayList<>(); + + private DiskCleanupRuleMock addFile(Path path, Priority priority) throws IOException { + PosixFileAttributes attributes = Files.getFileAttributeView(path, PosixFileAttributeView.class).readAttributes(); + pfa.add(new PrioritizedFileAttributes(new FileAttributes(path, attributes), priority)); + return this; + } + + @Override + public Collection<PrioritizedFileAttributes> prioritize() { + return Collections.unmodifiableList(pfa); + } + } + + private static class DiskCleanupTester { + private final FileSystem fileSystem = TestFileSystem.create(); + private final Set<String> files = new HashSet<>(); + + private Path path(String path) { + return fileSystem.getPath(path); + } + + private Path createFile(String pathStr, int size) throws IOException { + Path path = path(pathStr); + Files.createDirectories(path.getParent()); + Files.write(path, new byte[size]); + files.add(path.toString()); + return path; + } + + private void assertAllFilesExistExcept(String... deletedPaths) { + Set<String> actual = FileFinder.files(path("/")).stream().map(fa -> fa.path().toString()).collect(Collectors.toSet()); + Set<String> expected = new HashSet<>(files); + expected.removeAll(Set.of(deletedPaths)); + assertEquals(expected, actual); + } + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java new file mode 100644 index 00000000000..ae2d3af4f21 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/disk/LinearCleanupRuleTest.java @@ -0,0 +1,55 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.disk; + +import org.junit.Test; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder.FileAttributes; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.PrioritizedFileAttributes; +import static com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanupRule.Priority; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +/** + * @author freva + */ +public class LinearCleanupRuleTest { + + @Test + public void basic() { + assertRule(Map.of(), Priority.LOWEST, Priority.HIGHEST); + + assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.LOW, 1.0, Priority.LOW), Priority.LOW, Priority.LOW); + assertRule(Map.of(0.0, Priority.LOW, 0.5, Priority.MEDIUM, 1.0, Priority.MEDIUM), Priority.LOW, Priority.MEDIUM); + + assertRule(Map.of( + -5.0, Priority.LOW, + 0.0, Priority.LOW, + 0.2, Priority.LOW, + 0.35, Priority.MEDIUM, + 0.65, Priority.MEDIUM, + 0.8, Priority.HIGH, + 1.0, Priority.HIGH, + 5.0, Priority.HIGH), + Priority.LOW, Priority.HIGH); + } + + @Test(expected = IllegalArgumentException.class) + public void fail_if_high_priority_lower_than_low() { + assertRule(Map.of(), Priority.HIGHEST, Priority.LOWEST); + } + + private static void assertRule(Map<Double, Priority> expectedPriorities, Priority low, Priority high) { + Map<FileAttributes, Double> fileAttributesByScore = expectedPriorities.keySet().stream() + .collect(Collectors.toMap(score -> mock(FileAttributes.class), score -> score)); + LinearCleanupRule rule = new LinearCleanupRule( + () -> List.copyOf(fileAttributesByScore.keySet()), fileAttributesByScore::get, low, high); + + Map<Double, Priority> actualPriorities = rule.prioritize().stream() + .collect(Collectors.toMap(pfa -> fileAttributesByScore.get(pfa.fileAttributes()), PrioritizedFileAttributes::priority)); + assertEquals(expectedPriorities, actualPriorities); + } +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index b895300d7b1..dc1c0466d39 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -78,14 +78,12 @@ public class NodeAgentImplTest { NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(187500000000L)); nodeAgent.doConverge(context); verify(dockerOperations, never()).removeContainer(eq(context), any()); verify(orchestrator, never()).suspend(any(String.class)); verify(dockerOperations, never()).pullImageAsyncIfNeeded(any()); - verify(storageMaintainer, never()).removeOldFilesFromNode(eq(context)); final InOrder inOrder = inOrder(dockerOperations, orchestrator, nodeRepository); // TODO: Verify this isn't run unless 1st time @@ -104,11 +102,10 @@ public class NodeAgentImplTest { NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(217432719360L)); nodeAgent.doConverge(context); - verify(storageMaintainer, times(1)).removeOldFilesFromNode(eq(context)); + verify(storageMaintainer, times(1)).cleanDiskIfFull(eq(context)); } @Test @@ -122,7 +119,6 @@ public class NodeAgentImplTest { NodeAgentContext context = createContext(node); NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(187500000000L)); nodeAgent.doConverge(context); inOrder.verify(dockerOperations, never()).startServices(eq(context)); @@ -158,7 +154,6 @@ public class NodeAgentImplTest { when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); nodeAgent.doConverge(context); @@ -191,7 +186,6 @@ public class NodeAgentImplTest { when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); nodeAgent.doConverge(context); @@ -214,7 +208,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true); - when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); InOrder inOrder = inOrder(orchestrator, dockerOperations); @@ -262,7 +255,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(dockerOperations.pullImageAsyncIfNeeded(any())).thenReturn(true); - when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); nodeAgent.doConverge(firstContext); NodeAgentContext secondContext = createContext(specBuilder.memoryGb(20).build()); @@ -323,7 +315,6 @@ public class NodeAgentImplTest { when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); doThrow(new ConvergenceException("Connection refused")).doNothing() .when(healthChecker).verifyHealth(eq(context)); @@ -495,7 +486,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, false); when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); nodeAgent.doConverge(context); @@ -517,7 +507,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true); when(nodeRepository.getOptionalNode(eq(hostName))).thenReturn(Optional.of(node)); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); final InOrder inOrder = inOrder(orchestrator, dockerOperations, nodeRepository); doThrow(new RuntimeException("Failed 1st time")) @@ -553,7 +542,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = spy(makeNodeAgent(null, false)); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); doThrow(new DockerException("Failed to set up network")).doNothing().when(dockerOperations).startContainer(eq(context)); try { @@ -591,7 +579,6 @@ public class NodeAgentImplTest { when(nodeRepository.getOptionalNode(hostName)).thenReturn(Optional.of(node)); when(dockerOperations.pullImageAsyncIfNeeded(eq(dockerImage))).thenReturn(false); - when(storageMaintainer.getDiskUsageFor(eq(context))).thenReturn(Optional.of(201326592000L)); nodeAgent.doConverge(context); @@ -629,8 +616,6 @@ public class NodeAgentImplTest { NodeAgentContext context = createContext(specBuilder.build()); NodeAgentImpl nodeAgent = makeNodeAgent(null, false, Duration.ofSeconds(30)); - when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); - InOrder inOrder = inOrder(orchestrator, dockerOperations); ConvergenceException healthCheckException = new ConvergenceException("Not yet up"); @@ -685,8 +670,6 @@ public class NodeAgentImplTest { NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true); - when(storageMaintainer.getDiskUsageFor(any())).thenReturn(Optional.of(201326592000L)); - InOrder inOrder = inOrder(orchestrator, dockerOperations); nodeAgent.doConverge(context); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java new file mode 100644 index 00000000000..bcf3651fdfe --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/DiskSizeTest.java @@ -0,0 +1,25 @@ +package com.yahoo.vespa.hosted.node.admin.task.util.file; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +public class DiskSizeTest { + + @Test + public void bytes_to_display_count_test() { + assertEquals("-1 bytes", DiskSize.of(-1).asString()); + assertEquals("123 bytes", DiskSize.of(123).asString()); + assertEquals("1 kB", DiskSize.of(1_000).asString()); + assertEquals("15 MB", DiskSize.of(15_000_000).asString()); + assertEquals("123 GB", DiskSize.of(123_456_789_012L).asString()); + assertEquals("988 TB", DiskSize.of(987_654_321_098_765L).asString()); + assertEquals("987.7 TB", DiskSize.of(987_654_321_098_765L).asString(1)); + assertEquals("987.65 TB", DiskSize.of(987_654_321_098_765L).asString(2)); + assertEquals("2 PB", DiskSize.of(2_000_000_000_000_000L).asString()); + assertEquals("9 EB", DiskSize.of(Long.MAX_VALUE).asString()); + } +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java index 197077159d1..26897839b32 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/FileFinderTest.java @@ -2,15 +2,15 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; import org.junit.experimental.runners.Enclosed; -import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -40,8 +40,8 @@ import static org.mockito.Mockito.when; public class FileFinderTest { public static class GeneralLogicTests { - @Rule - public TemporaryFolder folder = new TemporaryFolder(); + + private final FileSystem fileSystem = TestFileSystem.create(); @Test public void all_files_non_recursive() { @@ -61,6 +61,27 @@ public class FileFinderTest { } @Test + public void all_files_recursive_with_prune_relative() { + assertFileHelper(FileFinder.files(testRoot()).prune(fileSystem.getPath("test")), + + of("file-1.json", "test.json", "test.txt"), + of("test", "test/file.txt", "test/data.json", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); + } + + @Test + public void all_files_recursive_with_prune_absolute() { + assertFileHelper(FileFinder.files(testRoot()).prune(testRoot().resolve("test/subdir-1")), + + of("file-1.json", "test.json", "test.txt", "test/file.txt", "test/data.json"), + of("test", "test/subdir-1", "test/subdir-1/test", "test/subdir-2")); + } + + @Test(expected = IllegalArgumentException.class) + public void throws_if_prune_path_not_under_base_path() { + FileFinder.files(Paths.get("/some/path")).prune(Paths.get("/other/path")); + } + + @Test public void with_file_filter_recursive() { assertFileHelper(FileFinder.files(testRoot()) .match(FileFinder.nameEndsWith(".json")), @@ -111,6 +132,7 @@ public class FileFinderTest { @Before public void setup() throws IOException { Path root = testRoot(); + Files.createDirectories(root); Files.createFile(root.resolve("file-1.json")); Files.createFile(root.resolve("test.json")); @@ -127,7 +149,7 @@ public class FileFinderTest { } private Path testRoot() { - return folder.getRoot().toPath(); + return fileSystem.getPath("/file-finder"); } private void assertFileHelper(FileFinder fileFinder, Set<String> expectedList, Set<String> expectedContentsAfterDelete) { diff --git a/node-repository/src/main/config/node-repository.xml b/node-repository/src/main/config/node-repository.xml index 186f052a274..90e98d11ac0 100644 --- a/node-repository/src/main/config/node-repository.xml +++ b/node-repository/src/main/config/node-repository.xml @@ -9,11 +9,11 @@ <component id="com.yahoo.config.provision.NodeFlavors" bundle="config-provisioning" /> <component id="com.yahoo.vespa.hosted.provision.provisioning.ProvisionServiceProvider" class="com.yahoo.vespa.hosted.provision.provisioning.EmptyProvisionServiceProvider" bundle="node-repository"/> -<handler id="com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler" bundle="node-repository"> +<handler id="com.yahoo.vespa.hosted.provision.restapi.NodesV2ApiHandler" bundle="node-repository"> <binding>http://*/nodes/v2/*</binding> </handler> -<handler id="com.yahoo.vespa.hosted.provision.restapi.v2.LoadBalancersApiHandler" bundle="node-repository"> +<handler id="com.yahoo.vespa.hosted.provision.restapi.LoadBalancersV1ApiHandler" bundle="node-repository"> <binding>http://*/loadbalancers/v1/*</binding> </handler> diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 6044bce7610..7045a47acf3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -38,7 +38,7 @@ import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; import com.yahoo.vespa.hosted.provision.provisioning.DockerImages; import com.yahoo.vespa.hosted.provision.provisioning.FirmwareChecks; -import com.yahoo.vespa.hosted.provision.restapi.v2.NotFoundException; +import com.yahoo.vespa.hosted.provision.restapi.NotFoundException; import java.time.Clock; import java.time.Duration; @@ -82,7 +82,8 @@ import java.util.stream.Stream; // 1) (new) | deprovisioned - > provisioned -> (dirty ->) ready -> reserved -> active -> inactive -> dirty -> ready // 2) inactive -> reserved | parked // 3) reserved -> dirty -// 3) * -> failed | parked -> dirty | active | deprovisioned +// 4) * -> failed | parked -> dirty | active | deprovisioned +// 5) deprovisioned -> (forgotten) // Nodes have an application assigned when in states reserved, active and inactive. // Nodes might have an application assigned in dirty. public class NodeRepository extends AbstractComponent { @@ -644,6 +645,13 @@ public class NodeRepository extends AbstractComponent { } } + /** Forgets a deprovisioned node. This removes all traces of the node in the node repository. */ + public void forget(Node node) { + if (node.state() != State.deprovisioned) + throw new IllegalArgumentException(node + " must be deprovisioned before it can be forgotten"); + db.removeNodes(List.of(node)); + } + /** * Throws if the given node cannot be removed. Removal is allowed if: * - Tenant node: node is unallocated @@ -778,11 +786,10 @@ public class NodeRepository extends AbstractComponent { public Zone zone() { return zone; } /** Create a lock which provides exclusive rights to making changes to the given application */ - // TODO(mpolden): Make this delegate to CuratorDatabaseClient#lockConfig instead - public Mutex lock(ApplicationId application) { return db.lock(application); } + public Mutex lock(ApplicationId application) { return db.legacyLock(application); } /** Create a lock with a timeout which provides exclusive rights to making changes to the given application */ - public Mutex lock(ApplicationId application, Duration timeout) { return db.lock(application, timeout); } + public Mutex lock(ApplicationId application, Duration timeout) { return db.legacyLock(application, timeout); } /** Create a lock which provides exclusive rights to modifying unallocated nodes */ public Mutex lockUnallocated() { return db.lockInactive(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java index e90a5f132ed..3a3d3e4ec2e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Application.java @@ -39,6 +39,10 @@ public class Application { public Map<ClusterSpec.Id, Cluster> clusters() { return clusters; } + public Optional<Cluster> cluster(ClusterSpec.Id id) { + return Optional.ofNullable(clusters.get(id)); + } + public Application with(Cluster cluster) { Map<ClusterSpec.Id, Cluster> clusters = new HashMap<>(this.clusters); clusters.put(cluster.id(), cluster); @@ -52,22 +56,12 @@ public class Application { public Application withClusterLimits(ClusterSpec.Id id, ClusterResources min, ClusterResources max) { Cluster cluster = clusters.get(id); if (cluster == null) - cluster = new Cluster(id, min, max, Optional.empty()); + cluster = new Cluster(id, min, max, Optional.empty(), Optional.empty()); else cluster = cluster.withLimits(min, max); return with(cluster); } - /** - * Returns an application with the given target for the given cluster, - * if it exists and the target is within the bounds - */ - public Application withClusterTarget(ClusterSpec.Id id, ClusterResources target) { - Cluster cluster = clusters.get(id); - if (cluster == null) return this; - return with(cluster.withTarget(target)); - } - @Override public int hashCode() { return id.hashCode(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index 914bc6c5a48..15a5545bc2c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -20,17 +20,19 @@ public class Cluster { private final ClusterSpec.Id id; private final ClusterResources min, max; + private final Optional<ClusterResources> suggested; private final Optional<ClusterResources> target; public Cluster(ClusterSpec.Id id, ClusterResources minResources, ClusterResources maxResources, + Optional<ClusterResources> suggestedResources, Optional<ClusterResources> targetResources) { this.id = Objects.requireNonNull(id); this.min = Objects.requireNonNull(minResources); this.max = Objects.requireNonNull(maxResources); + this.suggested = Objects.requireNonNull(suggestedResources); Objects.requireNonNull(targetResources); - if (targetResources.isPresent() && ! targetResources.get().isWithin(minResources, maxResources)) this.target = Optional.empty(); else @@ -47,20 +49,27 @@ public class Cluster { /** * Returns the computed resources (between min and max, inclusive) this cluster should - * have allocated at the moment, or empty if the system currently have no opinion on this. + * have allocated at the moment (whether or not it actually has it), + * or empty if the system currently has no target. */ public Optional<ClusterResources> targetResources() { return target; } + /** + * The suggested size of this cluster, which may or may not be within the min and max limits, + * or empty if there is currently no suggestion. + */ + public Optional<ClusterResources> suggestedResources() { return suggested; } + public Cluster withLimits(ClusterResources min, ClusterResources max) { - return new Cluster(id, min, max, target); + return new Cluster(id, min, max, suggested, target); } - public Cluster withTarget(ClusterResources target) { - return new Cluster(id, min, max, Optional.of(target)); + public Cluster withSuggested(Optional<ClusterResources> suggested) { + return new Cluster(id, min, max, suggested, target); } - public Cluster withoutTarget() { - return new Cluster(id, min, max, Optional.empty()); + public Cluster withTarget(Optional<ClusterResources> target) { + return new Cluster(id, min, max, suggested, target); } public NodeResources capAtLimits(NodeResources resources) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 447d3494fbc..6dca9c9a796 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -31,7 +31,6 @@ public class Autoscaler { - Scale group size - Consider taking spikes/variance into account - Measure observed regulation lag (startup+redistribution) and take it into account when deciding regulation observation window - - Test AutoscalingMaintainer - Scale by performance not just load+cost */ @@ -57,12 +56,31 @@ public class Autoscaler { } /** - * Autoscale a cluster + * Suggest a scaling of a cluster. This returns a better allocation (if found) + * without taking min and max limits into account. * * @param clusterNodes the list of all the active nodes in a cluster * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time */ - public Optional<AllocatableClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes) { + public Optional<ClusterResources> suggest(Cluster cluster, List<Node> clusterNodes) { + return autoscale(cluster, clusterNodes, false) + .map(AllocatableClusterResources::toAdvertisedClusterResources); + + } + + /** + * Autoscale a cluster. This returns a better allocation (if found) inside the min and max limits. + * + * @param clusterNodes the list of all the active nodes in a cluster + * @return a new suggested allocation for this cluster, or empty if it should not be rescaled at this time + */ + public Optional<ClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes) { + if (cluster.minResources().equals(cluster.maxResources())) return Optional.empty(); // Shortcut + return autoscale(cluster, clusterNodes, true) + .map(AllocatableClusterResources::toAdvertisedClusterResources); + } + + private Optional<AllocatableClusterResources> autoscale(Cluster cluster, List<Node> clusterNodes, boolean respectLimits) { if (unstable(clusterNodes)) return Optional.empty(); ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); @@ -76,7 +94,8 @@ public class Autoscaler { memoryLoad.get(), diskLoad.get(), currentAllocation, - cluster); + cluster, + respectLimits); if (bestAllocation.isEmpty()) return Optional.empty(); if (similar(bestAllocation.get(), currentAllocation)) return Optional.empty(); return bestAllocation; @@ -84,12 +103,14 @@ public class Autoscaler { private Optional<AllocatableClusterResources> findBestAllocation(double cpuLoad, double memoryLoad, double diskLoad, AllocatableClusterResources currentAllocation, - Cluster cluster) { + Cluster cluster, boolean respectLimits) { Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); - for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation, cluster); i.hasNext(); ) { + for (ResourceIterator i = new ResourceIterator(cpuLoad, memoryLoad, diskLoad, currentAllocation, cluster, respectLimits); + i.hasNext(); ) { Optional<AllocatableClusterResources> allocatableResources = toAllocatableResources(i.next(), currentAllocation.clusterType(), - cluster); + cluster, + respectLimits); if (allocatableResources.isEmpty()) continue; if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) bestAllocation = allocatableResources; @@ -119,9 +140,10 @@ public class Autoscaler { */ private Optional<AllocatableClusterResources> toAllocatableResources(ClusterResources resources, ClusterSpec.Type clusterType, - Cluster cluster) { + Cluster cluster, + boolean respectLimits) { NodeResources nodeResources = resources.nodeResources(); - if ( ! cluster.minResources().equals(cluster.maxResources())) // enforce application limits unless suggest mode + if (respectLimits) nodeResources = cluster.capAtLimits(nodeResources); nodeResources = nodeResourceLimits.enlargeToLegal(nodeResources, clusterType); // enforce system limits diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java index b7d5995884e..207eecc1871 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ResourceIterator.java @@ -11,8 +11,8 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster; */ public class ResourceIterator { - // Configured min and max nodes for suggestions for apps which have not activated autoscaling - private static final int minimumNodes = 3; // Since this is with redundancy it cannot be lower than 2 + // The min and max nodes to consider when not using application supplied limits + private static final int minimumNodes = 3; // Since this number includes redundancy it cannot be lower than 2 private static final int maximumNodes = 150; // When a query is issued on a node the cost is the sum of a fixed cost component and a cost component @@ -22,6 +22,7 @@ public class ResourceIterator { // Prescribed state private final Cluster cluster; + private final boolean respectLimits; // Observed state private final AllocatableClusterResources allocation; @@ -39,10 +40,12 @@ public class ResourceIterator { public ResourceIterator(double cpuLoad, double memoryLoad, double diskLoad, AllocatableClusterResources currentAllocation, - Cluster cluster) { + Cluster cluster, + boolean respectLimits) { this.cpuLoad = cpuLoad; this.memoryLoad = memoryLoad; this.diskLoad = diskLoad; + this.respectLimits = respectLimits; // ceil: If the division does not produce a whole number we assume some node is missing groupSize = (int)Math.ceil((double)currentAllocation.nodes() / currentAllocation.groups()); @@ -64,11 +67,6 @@ public class ResourceIterator { currentNodes -= nodeIncrement; } - /** If autoscaling is not enabled (meaning max and min resources are the same), we want to suggest */ - private boolean suggestMode() { - return cluster.minResources().equals(cluster.maxResources()); - } - public ClusterResources next() { ClusterResources next = resourcesWith(currentNodes); currentNodes += nodeIncrement; @@ -80,13 +78,13 @@ public class ResourceIterator { } private int minNodes() { - if (suggestMode()) return minimumNodes; + if ( ! respectLimits) return minimumNodes; if (singleGroupMode) return cluster.minResources().nodes(); return Math.max(cluster.minResources().nodes(), cluster.minResources().groups() * groupSize ); } private int maxNodes() { - if (suggestMode()) return maximumNodes; + if ( ! respectLimits) return maximumNodes; if (singleGroupMode) return cluster.maxResources().nodes(); return Math.min(cluster.maxResources().nodes(), cluster.maxResources().groups() * groupSize ); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 44dd023677c..b58fd3f67f1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -5,7 +5,7 @@ import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Deployer; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; @@ -63,7 +63,7 @@ public abstract class ApplicationMaintainer extends Maintainer { */ protected void deploy(ApplicationId application) { if (pendingDeployments.addIfAbsent(application)) { // Avoid queuing multiple deployments for same application - log.log(LogLevel.INFO, application + " will be deployed, last deploy time " + + log.log(Level.INFO, application + " will be deployed, last deploy time " + getLastDeployTime(application)); deploymentExecutor.execute(() -> deployWithLock(application)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index d17577af135..94487547fab 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -3,13 +3,16 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.collections.Pair; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.NodeResources; import com.yahoo.jdisc.Metric; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; @@ -34,7 +37,6 @@ public class AutoscalingMaintainer extends Maintainer { private final Autoscaler autoscaler; private final Deployer deployer; private final Metric metric; - private final Map<Pair<ApplicationId, ClusterSpec.Id>, Instant> lastLogged = new HashMap<>(); public AutoscalingMaintainer(NodeRepository nodeRepository, HostResourcesCalculator hostResourcesCalculator, @@ -67,43 +69,30 @@ public class AutoscalingMaintainer extends Maintainer { List<Node> clusterNodes, MaintenanceDeployment deployment) { Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)); - Cluster cluster = application.clusters().get(clusterId); - if (cluster == null) return; // no information on limits for this cluster - Optional<AllocatableClusterResources> target = autoscaler.autoscale(cluster, clusterNodes); - if (target.isEmpty()) return; // current resources are fine - - if (cluster.minResources().equals(cluster.maxResources())) { // autoscaling is deactivated - logAutoscaling("Scaling suggestion for ", target.get(), applicationId, clusterId, clusterNodes); - } - else { - logAutoscaling("Autoscaling ", target.get(), applicationId, clusterId, clusterNodes); - autoscaleTo(target.get(), clusterId, application, deployment); + Optional<Cluster> cluster = application.cluster(clusterId); + if (cluster.isEmpty()) return; + Optional<ClusterResources> target = autoscaler.autoscale(cluster.get(), clusterNodes); + if ( ! cluster.get().targetResources().equals(target)) + applications().put(application.with(cluster.get().withTarget(target)), deployment.applicationLock().get()); + if (target.isPresent()) { + logAutoscaling(target.get(), applicationId, clusterId, clusterNodes); + deployment.activate(); } } - private void autoscaleTo(AllocatableClusterResources target, - ClusterSpec.Id clusterId, - Application application, - MaintenanceDeployment deployment) { - nodeRepository().applications().put(application.withClusterTarget(clusterId, target.toAdvertisedClusterResources()), - deployment.applicationLock().get()); - deployment.activate(); + private Applications applications() { + return nodeRepository().applications(); } - private void logAutoscaling(String prefix, - AllocatableClusterResources target, + private void logAutoscaling(ClusterResources target, ApplicationId application, ClusterSpec.Id clusterId, List<Node> clusterNodes) { - Instant lastLogTime = lastLogged.get(new Pair<>(application, clusterId)); - if (lastLogTime != null && lastLogTime.isAfter(nodeRepository().clock().instant().minus(Duration.ofHours(1)))) return; - int currentGroups = (int)clusterNodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); ClusterSpec.Type clusterType = clusterNodes.get(0).allocation().get().membership().cluster().type(); - log.info(prefix + application + " " + clusterType + " " + clusterId + ":" + + log.info("Autoscaling " + application + " " + clusterType + " " + clusterId + ":" + "\nfrom " + toString(clusterNodes.size(), currentGroups, clusterNodes.get(0).flavor().resources()) + - "\nto " + toString(target.nodes(), target.groups(), target.advertisedResources())); - lastLogged.put(new Pair<>(application, clusterId), nodeRepository().clock().instant()); + "\nto " + toString(target.nodes(), target.groups(), target.nodeResources())); } private String toString(int nodes, int groups, NodeResources resources) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java index d08811e9255..b4c0d469272 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityReportMaintainer.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -42,7 +42,7 @@ public class CapacityReportMaintainer extends Maintainer { CapacityChecker capacityChecker = new CapacityChecker(this.nodeRepository); List<Node> overcommittedHosts = capacityChecker.findOvercommittedHosts(); if (overcommittedHosts.size() != 0) { - log.log(LogLevel.WARNING, String.format("%d nodes are overcommitted! [ %s ]", overcommittedHosts.size(), + log.log(Level.WARNING, String.format("%d nodes are overcommitted! [ %s ]", overcommittedHosts.size(), overcommittedHosts.stream().map(Node::hostname).collect(Collectors.joining(", ")))); } metric.set("overcommittedHosts", overcommittedHosts.size(), null); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java index 4d04409aaf0..932c435221c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/InfrastructureProvisioner.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.InfraDeployer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; @@ -29,7 +29,7 @@ public class InfrastructureProvisioner extends Maintainer { try { infraDeployer.activateAllSupportedInfraApplications(true); } catch (RuntimeException e) { - logger.log(LogLevel.INFO, "Failed to deploy supported infrastructure applications, " + + logger.log(Level.INFO, "Failed to deploy supported infrastructure applications, " + "will sleep 30s before propagating failure, to allow inspection of zk", e.getMessage()); try { Thread.sleep(30_000); } catch (InterruptedException ignored) { } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index d7a93288d2d..55152f52744 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; @@ -79,7 +79,7 @@ public class LoadBalancerExpirer extends Maintainer { } }); if (!failed.isEmpty()) { - log.log(LogLevel.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s", + log.log(Level.WARNING, String.format("Failed to remove %d load balancers: %s, retrying in %s", failed.size(), failed.stream() .map(LoadBalancerId::serializedForm) @@ -107,7 +107,7 @@ public class LoadBalancerExpirer extends Maintainer { } }); if (!failed.isEmpty()) { - log.log(LogLevel.WARNING, String.format("Failed to remove reals from %d load balancers: %s, retrying in %s", + log.log(Level.WARNING, String.format("Failed to remove reals from %d load balancers: %s, retrying in %s", failed.size(), failed.stream() .map(LoadBalancerId::serializedForm) @@ -120,7 +120,7 @@ public class LoadBalancerExpirer extends Maintainer { /** Apply operation to all load balancers that exist in given state, while holding lock */ private void withLoadBalancersIn(LoadBalancer.State state, Consumer<LoadBalancer> operation) { for (var id : db.readLoadBalancerIds()) { - try (var lock = db.lockConfig(id.application())) { + try (var lock = db.lock(id.application())) { var loadBalancer = db.readLoadBalancer(id); if (loadBalancer.isEmpty()) continue; // Load balancer was removed during loop if (loadBalancer.get().state() != state) continue; // Wrong state diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java index 8e1cd801a15..3319db33730 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.TransientException; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -83,12 +83,12 @@ class MaintenanceDeployment implements Closeable { return true; } catch (TransientException e) { metric.add("maintenanceDeployment.transientFailure", 1, metric.createContext(Map.of())); - log.log(LogLevel.INFO, "Failed to maintenance deploy " + application + " with a transient error: " + + log.log(Level.INFO, "Failed to maintenance deploy " + application + " with a transient error: " + Exceptions.toMessageString(e)); return false; } catch (RuntimeException e) { metric.add("maintenanceDeployment.failure", 1, metric.createContext(Map.of())); - log.log(LogLevel.WARNING, "Exception on maintenance deploy of " + application, e); + log.log(Level.WARNING, "Exception on maintenance deploy of " + application, e); return false; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index b0b8f7d8d2c..cffbfec8f4b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.HostLivenessTracker; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TransientException; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.applicationmodel.ServiceInstance; @@ -366,7 +366,7 @@ public class NodeFailer extends Maintainer { deployment.get().activate(); return true; } catch (TransientException e) { - log.log(LogLevel.INFO, "Failed to redeploy " + node.allocation().get().owner() + + log.log(Level.INFO, "Failed to redeploy " + node.allocation().get().owner() + " with a transient error, will be retried by application maintainer: " + Exceptions.toMessageString(e)); return true; } catch (RuntimeException e) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 054c273dc99..9e954e0a1dd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -52,6 +52,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Rebalancer rebalancer; private final NodeMetricsDbMaintainer nodeMetricsDbMaintainer; private final AutoscalingMaintainer autoscalingMaintainer; + private final ScalingSuggestionsMaintainer scalingSuggestionsMaintainer; @SuppressWarnings("unused") @Inject @@ -92,6 +93,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { rebalancer = new Rebalancer(deployer, nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), provisionServiceProvider.getHostProvisioner(), metric, clock, defaults.rebalancerInterval); nodeMetricsDbMaintainer = new NodeMetricsDbMaintainer(nodeRepository, nodeMetrics, nodeMetricsDb, defaults.nodeMetricsCollectionInterval); autoscalingMaintainer = new AutoscalingMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, deployer, metric, defaults.autoscalingInterval); + scalingSuggestionsMaintainer = new ScalingSuggestionsMaintainer(nodeRepository, provisionServiceProvider.getHostResourcesCalculator(), nodeMetricsDb, defaults.scalingSuggestionsInterval); // The DuperModel is filled with infrastructure applications by the infrastructure provisioner, so explicitly run that now infrastructureProvisioner.maintainButThrowOnException(); @@ -118,6 +120,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { rebalancer.deconstruct(); nodeMetricsDbMaintainer.deconstruct(); autoscalingMaintainer.deconstruct(); + scalingSuggestionsMaintainer.deconstruct(); } private static Optional<NodeFailer.ThrottlePolicy> throttlePolicyFromEnv() { @@ -160,6 +163,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final Duration rebalancerInterval; private final Duration nodeMetricsCollectionInterval; private final Duration autoscalingInterval; + private final Duration scalingSuggestionsInterval; private final NodeFailer.ThrottlePolicy throttlePolicy; @@ -182,6 +186,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { rebalancerInterval = Duration.ofMinutes(40); nodeMetricsCollectionInterval = Duration.ofMinutes(1); autoscalingInterval = Duration.ofMinutes(5); + scalingSuggestionsInterval = Duration.ofMinutes(31); if (zone.environment().equals(Environment.prod) && ! zone.system().isCd()) { inactiveExpiry = Duration.ofHours(4); // enough time for the application owner to discover and redeploy diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java new file mode 100644 index 00000000000..6296543aea4 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -0,0 +1,82 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Applications; +import com.yahoo.vespa.hosted.provision.applications.Cluster; +import com.yahoo.vespa.hosted.provision.autoscale.AllocatableClusterResources; +import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Maintainer computing scaling suggestions for all clusters + * + * @author bratseth + */ +public class ScalingSuggestionsMaintainer extends Maintainer { + + private final Autoscaler autoscaler; + + public ScalingSuggestionsMaintainer(NodeRepository nodeRepository, + HostResourcesCalculator hostResourcesCalculator, + NodeMetricsDb metricsDb, + Duration interval) { + super(nodeRepository, interval); + this.autoscaler = new Autoscaler(hostResourcesCalculator, metricsDb, nodeRepository); + } + + @Override + protected void maintain() { + if ( ! nodeRepository().zone().environment().isProduction()) return; + + activeNodesByApplication().forEach((applicationId, nodes) -> suggest(applicationId, nodes)); + } + + private void suggest(ApplicationId application, List<Node> applicationNodes) { + nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> + suggest(application, clusterId, clusterNodes)); + } + + private Applications applications() { + return nodeRepository().applications(); + } + + private void suggest(ApplicationId applicationId, + ClusterSpec.Id clusterId, + List<Node> clusterNodes) { + Application application = applications().get(applicationId).orElse(new Application(applicationId)); + Optional<Cluster> cluster = application.cluster(clusterId); + if (cluster.isEmpty()) return; + Optional<ClusterResources> suggestion = autoscaler.suggest(cluster.get(), clusterNodes); + try (Mutex lock = nodeRepository().lock(applicationId)) { + applications().get(applicationId).ifPresent(a -> storeSuggestion(suggestion, clusterId, a, lock)); + } + } + + private void storeSuggestion(Optional<ClusterResources> suggestion, + ClusterSpec.Id clusterId, + Application application, + Mutex lock) { + Optional<Cluster> cluster = application.cluster(clusterId); + if (cluster.isEmpty()) return; + applications().put(application.with(cluster.get().withSuggested(suggestion)), lock); + } + + private Map<ClusterSpec.Id, List<Node>> nodesByCluster(List<Node> applicationNodes) { + return applicationNodes.stream().collect(Collectors.groupingBy(n -> n.allocation().get().membership().cluster().id())); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index f514cc20fe8..b1453a9729a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -37,6 +37,7 @@ public class ApplicationSerializer { private static final String clustersKey = "clusters"; private static final String minResourcesKey = "min"; private static final String maxResourcesKey = "max"; + private static final String suggestedResourcesKey = "suggested"; private static final String targetResourcesKey = "target"; private static final String nodesKey = "nodes"; private static final String groupsKey = "groups"; @@ -81,6 +82,7 @@ public class ApplicationSerializer { private static void toSlime(Cluster cluster, Cursor clusterObject) { toSlime(cluster.minResources(), clusterObject.setObject(minResourcesKey)); toSlime(cluster.maxResources(), clusterObject.setObject(maxResourcesKey)); + cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject(suggestedResourcesKey))); cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject(targetResourcesKey))); } @@ -88,6 +90,7 @@ public class ApplicationSerializer { return new Cluster(ClusterSpec.Id.from(id), clusterResourcesFromSlime(clusterObject.field(minResourcesKey)), clusterResourcesFromSlime(clusterObject.field(maxResourcesKey)), + optionalClusterResourcesFromSlime(clusterObject.field(suggestedResourcesKey)), optionalClusterResourcesFromSlime(clusterObject.field(targetResourcesKey))); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index af4e054579e..347c66d5426 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -10,7 +10,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -76,7 +76,7 @@ public class CuratorDatabaseClient { private final NodeSerializer nodeSerializer; private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); - private final CuratorDatabase curatorDatabase; + private final CuratorDatabase db; private final Clock clock; private final Zone zone; private final CuratorCounter provisionIndexCounter; @@ -84,27 +84,27 @@ public class CuratorDatabaseClient { public CuratorDatabaseClient(NodeFlavors flavors, Curator curator, Clock clock, Zone zone, boolean useCache) { this.nodeSerializer = new NodeSerializer(flavors); this.zone = zone; - this.curatorDatabase = new CuratorDatabase(curator, root, useCache); + this.db = new CuratorDatabase(curator, root, useCache); this.clock = clock; this.provisionIndexCounter = new CuratorCounter(curator, root.append("provisionIndexCounter").getAbsolute()); initZK(); } public List<HostName> cluster() { - return curatorDatabase.cluster(); + return db.cluster(); } private void initZK() { - curatorDatabase.create(root); + db.create(root); for (Node.State state : Node.State.values()) - curatorDatabase.create(toPath(state)); - curatorDatabase.create(applicationsPath); - curatorDatabase.create(inactiveJobsPath); - curatorDatabase.create(infrastructureVersionsPath); - curatorDatabase.create(osVersionsPath); - curatorDatabase.create(dockerImagesPath); - curatorDatabase.create(firmwareCheckPath); - curatorDatabase.create(loadBalancersPath); + db.create(toPath(state)); + db.create(applicationsPath); + db.create(inactiveJobsPath); + db.create(infrastructureVersionsPath); + db.create(osVersionsPath); + db.create(dockerImagesPath); + db.create(firmwareCheckPath); + db.create(loadBalancersPath); provisionIndexCounter.initialize(100); } @@ -113,7 +113,7 @@ public class CuratorDatabaseClient { */ public List<Node> addNodesInState(List<Node> nodes, Node.State expectedState) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); for (Node node : nodes) { if (node.state() != expectedState) throw new IllegalArgumentException(node + " is not in the " + expectedState + " state"); @@ -124,7 +124,7 @@ public class CuratorDatabaseClient { transaction.commit(); for (Node node : nodes) - log.log(LogLevel.INFO, "Added " + node); + log.log(Level.INFO, "Added " + node); return nodes; } @@ -139,12 +139,12 @@ public class CuratorDatabaseClient { for (Node node : nodes) { Path path = toPath(node.state(), node.hostname()); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.delete(path.getAbsolute())); } transaction.commit(); - nodes.forEach(node -> log.log(LogLevel.INFO, "Removed node " + node.hostname() + " in state " + node.state())); + nodes.forEach(node -> log.log(Level.INFO, "Removed node " + node.hostname() + " in state " + node.state())); } /** @@ -209,7 +209,7 @@ public class CuratorDatabaseClient { List<Node> writtenNodes = new ArrayList<>(nodes.size()); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); for (Node node : nodes) { Node newNode = new Node(node.id(), node.ipConfig(), node.hostname(), node.parentHostname(), node.flavor(), @@ -225,7 +225,7 @@ public class CuratorDatabaseClient { transaction.onCommitted(() -> { // schedule logging on commit of nodes which changed state for (Node node : nodes) { if (toState != node.state()) - log.log(LogLevel.INFO, agent + " moved " + node + " to " + toState + reason.map(s -> ": " + s).orElse("")); + log.log(Level.INFO, agent + " moved " + node + " to " + toState + reason.map(s -> ": " + s).orElse("")); } }); return writtenNodes; @@ -269,7 +269,7 @@ public class CuratorDatabaseClient { List<Node> nodes = new ArrayList<>(); if (states.length == 0) states = Node.State.values(); - CuratorDatabase.Session session = curatorDatabase.getSession(); + CuratorDatabase.Session session = db.getSession(); for (Node.State state : states) { for (String hostname : session.getChildren(toPath(state))) { Optional<Node> node = readNode(session, hostname, state); @@ -309,7 +309,7 @@ public class CuratorDatabaseClient { * If no states are given this returns the node if it is present in any state. */ public Optional<Node> readNode(String hostname, Node.State ... states) { - return readNode(curatorDatabase.getSession(), hostname, states); + return readNode(db.getSession(), hostname, states); } private Path toPath(Node.State nodeState) { return root.append(toDir(nodeState)); } @@ -323,21 +323,22 @@ public class CuratorDatabaseClient { } /** Creates and returns the path to the lock for this application */ - private Path lockPath(ApplicationId application) { + // TODO(mpolden): Remove when all config servers take the new lock + private Path legacyLockPath(ApplicationId application) { Path lockPath = CuratorDatabaseClient.lockPath .append(application.tenant().value()) .append(application.application().value()) .append(application.instance().value()); - curatorDatabase.create(lockPath); + db.create(lockPath); return lockPath; } - /** Creates and returns the path to the config server lock for this application */ - private Path configLockPath(ApplicationId application) { + /** Creates and returns the lock path for this application */ + private Path lockPath(ApplicationId application) { // This must match the lock path used by com.yahoo.vespa.config.server.application.TenantApplications Path lockPath = configLockPath.append(application.tenant().value()).append(application.serializedForm()); - curatorDatabase.create(lockPath); + db.create(lockPath); return lockPath; } @@ -358,18 +359,20 @@ public class CuratorDatabaseClient { /** Acquires the single cluster-global, reentrant lock for all non-active nodes */ public Lock lockInactive() { - return lock(lockPath.append("unallocatedLock"), defaultLockTimeout); + return db.lock(lockPath.append("unallocatedLock"), defaultLockTimeout); } /** Acquires the single cluster-global, reentrant lock for active nodes of this application */ - public Lock lock(ApplicationId application) { - return lock(application, defaultLockTimeout); + // TODO(mpolden): Remove when all config servers take the new lock + public Lock legacyLock(ApplicationId application) { + return legacyLock(application, defaultLockTimeout); } /** Acquires the single cluster-global, reentrant lock with the specified timeout for active nodes of this application */ - public Lock lock(ApplicationId application, Duration timeout) { + // TODO(mpolden): Remove when all config servers take the new lock + public Lock legacyLock(ApplicationId application, Duration timeout) { try { - return lock(lockPath(application), timeout); + return db.lock(legacyLockPath(application), timeout); } catch (UncheckedTimeoutException e) { throw new ApplicationLockException(e); @@ -377,7 +380,7 @@ public class CuratorDatabaseClient { } /** - * Acquires the single cluster-global, re-entrant config lock for given application. Note that this is the same lock + * Acquires the single cluster-global, re-entrant lock for given application. Note that this is the same lock * that configserver itself takes when modifying applications. * * This lock must be taken when writes to paths owned by this class may happen on both the configserver and @@ -390,20 +393,24 @@ public class CuratorDatabaseClient { * transaction. The config server then commits (writes) the transaction which may include operations that modify * data in paths owned by this class. */ - public Lock lockConfig(ApplicationId application) { + public Lock lock(ApplicationId application, Duration timeout) { try { - return lock(configLockPath(application), defaultLockTimeout); + return db.lock(lockPath(application), timeout); } catch (UncheckedTimeoutException e) { throw new ApplicationLockException(e); } } + public Lock lock(ApplicationId application) { + return lock(application, defaultLockTimeout); + } + // Applications ----------------------------------------------------------- public List<ApplicationId> readApplicationIds() { - return curatorDatabase.getChildren(applicationsPath).stream() - .map(path -> ApplicationId.fromSerializedForm(path)) - .collect(Collectors.toList()); + return db.getChildren(applicationsPath).stream() + .map(path -> ApplicationId.fromSerializedForm(path)) + .collect(Collectors.toList()); } public Optional<Application> readApplication(ApplicationId id) { @@ -411,15 +418,15 @@ public class CuratorDatabaseClient { } public void writeApplication(Application application, NestedTransaction transaction) { - curatorDatabase.newCuratorTransactionIn(transaction) - .add(createOrSet(applicationPath(application.id()), + db.newCuratorTransactionIn(transaction) + .add(createOrSet(applicationPath(application.id()), ApplicationSerializer.toJson(application))); } public void deleteApplication(ApplicationId application, NestedTransaction transaction) { - if (curatorDatabase.exists(applicationPath(application))) - curatorDatabase.newCuratorTransactionIn(transaction) - .add(CuratorOperations.delete(applicationPath(application).getAbsolute())); + if (db.exists(applicationPath(application))) + db.newCuratorTransactionIn(transaction) + .add(CuratorOperations.delete(applicationPath(application).getAbsolute())); } private Path applicationPath(ApplicationId id) { @@ -429,7 +436,7 @@ public class CuratorDatabaseClient { // Maintenance jobs ----------------------------------------------------------- public Lock lockMaintenanceJob(String jobName) { - return lock(lockPath.append("maintenanceJobLocks").append(jobName), defaultLockTimeout); + return db.lock(lockPath.append("maintenanceJobLocks").append(jobName), defaultLockTimeout); } public Set<String> readInactiveJobs() { @@ -445,14 +452,14 @@ public class CuratorDatabaseClient { public void writeInactiveJobs(Set<String> inactiveJobs) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(inactiveJobsPath.getAbsolute(), stringSetSerializer.toJson(inactiveJobs))); transaction.commit(); } public Lock lockInactiveJobs() { - return lock(lockPath.append("inactiveJobsLock"), defaultLockTimeout); + return db.lock(lockPath.append("inactiveJobsLock"), defaultLockTimeout); } // Infrastructure versions ----------------------------------------------------------- @@ -463,14 +470,14 @@ public class CuratorDatabaseClient { public void writeInfrastructureVersions(Map<NodeType, Version> infrastructureVersions) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(infrastructureVersionsPath.getAbsolute(), NodeTypeVersionsSerializer.toJson(infrastructureVersions))); transaction.commit(); } public Lock lockInfrastructureVersions() { - return lock(lockPath.append("infrastructureVersionsLock"), defaultLockTimeout); + return db.lock(lockPath.append("infrastructureVersionsLock"), defaultLockTimeout); } // OS versions ----------------------------------------------------------- @@ -481,14 +488,14 @@ public class CuratorDatabaseClient { public void writeOsVersions(Map<NodeType, Version> versions) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(osVersionsPath.getAbsolute(), OsVersionsSerializer.toJson(versions))); transaction.commit(); } public Lock lockOsVersions() { - return lock(lockPath.append("osVersionsLock"), defaultLockTimeout); + return db.lock(lockPath.append("osVersionsLock"), defaultLockTimeout); } // Docker images ----------------------------------------------------------- @@ -499,14 +506,14 @@ public class CuratorDatabaseClient { public void writeDockerImages(Map<NodeType, DockerImage> dockerImages) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(dockerImagesPath.getAbsolute(), NodeTypeDockerImagesSerializer.toJson(dockerImages))); transaction.commit(); } public Lock lockDockerImages() { - return lock(lockPath.append("dockerImagesLock"), defaultLockTimeout); + return db.lock(lockPath.append("dockerImagesLock"), defaultLockTimeout); } // Firmware checks ----------------------------------------------------------- @@ -516,7 +523,7 @@ public class CuratorDatabaseClient { byte[] data = after.map(instant -> Long.toString(instant.toEpochMilli()).getBytes()) .orElse(new byte[0]); NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(firmwareCheckPath.getAbsolute(), data)); transaction.commit(); } @@ -552,7 +559,7 @@ public class CuratorDatabaseClient { } public void writeLoadBalancers(Collection<LoadBalancer> loadBalancers, NestedTransaction transaction) { - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); loadBalancers.forEach(loadBalancer -> { curatorTransaction.add(createOrSet(loadBalancerPath(loadBalancer.id()), LoadBalancerSerializer.toJson(loadBalancer))); @@ -561,7 +568,7 @@ public class CuratorDatabaseClient { public void removeLoadBalancer(LoadBalancerId loadBalancer) { NestedTransaction transaction = new NestedTransaction(); - CuratorTransaction curatorTransaction = curatorDatabase.newCuratorTransactionIn(transaction); + CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.delete(loadBalancerPath(loadBalancer).getAbsolute())); transaction.commit(); } @@ -571,10 +578,10 @@ public class CuratorDatabaseClient { } private List<LoadBalancerId> readLoadBalancerIds(Predicate<LoadBalancerId> predicate) { - return curatorDatabase.getChildren(loadBalancersPath).stream() - .map(LoadBalancerId::fromSerializedForm) - .filter(predicate) - .collect(Collectors.toUnmodifiableList()); + return db.getChildren(loadBalancersPath).stream() + .map(LoadBalancerId::fromSerializedForm) + .filter(predicate) + .collect(Collectors.toUnmodifiableList()); } /** Returns a given number of unique provision indexes */ @@ -588,16 +595,12 @@ public class CuratorDatabaseClient { .collect(Collectors.toList()); } - private Lock lock(Path path, Duration timeout) { - return curatorDatabase.lock(path, timeout); - } - private <T> Optional<T> read(Path path, Function<byte[], T> mapper) { - return curatorDatabase.getData(path).filter(data -> data.length > 0).map(mapper); + return db.getData(path).filter(data -> data.length > 0).map(mapper); } private Transaction.Operation createOrSet(Path path, byte[] data) { - return curatorDatabase.exists(path) ? CuratorOperations.setData(path.getAbsolute(), data) + return db.exists(path) ? CuratorOperations.setData(path.getAbsolute(), data) : CuratorOperations.create(path.getAbsolute(), data); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java index 461b7346119..6d2c074ab36 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/DnsNameResolver.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.provision.persistence; import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import javax.naming.NamingException; import javax.naming.directory.Attribute; @@ -52,7 +52,7 @@ public class DnsNameResolver implements NameResolver { return InetAddresses.isInetAddress(hostname) ? Optional.empty() : Optional.of(hostname); } catch (UnknownHostException e) { // This is not an exceptional state hence the debug level - logger.log(LogLevel.DEBUG, "Unable to resolve ipaddress", e); + logger.log(Level.FINE, "Unable to resolve ipaddress", e); } return Optional.empty(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index fb83385920c..c67565db7a9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Provisioner; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -57,7 +57,7 @@ public class InfraDeployerImpl implements InfraDeployer { try { deployment.activate(); } catch (RuntimeException e) { - logger.log(LogLevel.INFO, "Failed to activate " + application, e); + logger.log(Level.INFO, "Failed to activate " + application, e); if (propagateException) { throw e; } @@ -101,7 +101,7 @@ public class InfraDeployerImpl implements InfraDeployer { prepare(); if (hostSpecs.isEmpty()) { - logger.log(LogLevel.DEBUG, "No nodes to provision for " + application.getCapacity().type() + ", removing application"); + logger.log(Level.FINE, "No nodes to provision for " + application.getCapacity().type() + ", removing application"); removeApplication(application.getApplicationId()); } else { NestedTransaction nestedTransaction = new NestedTransaction(); @@ -112,7 +112,7 @@ public class InfraDeployerImpl implements InfraDeployer { application.getApplicationId(), hostSpecs.stream().map(HostSpec::hostname).map(HostName::from).collect(Collectors.toList())); - logger.log(LogLevel.DEBUG, () -> generateActivationLogMessage(hostSpecs, application.getApplicationId())); + logger.log(Level.FINE, () -> generateActivationLogMessage(hostSpecs, application.getApplicationId())); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index f1fc6be837e..3532d989d85 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.exception.LoadBalancerServiceException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; @@ -52,7 +52,7 @@ public class LoadBalancerProvisioner { this.service = service; // Read and write all load balancers to make sure they are stored in the latest version of the serialization format for (var id : db.readLoadBalancerIds()) { - try (var lock = db.lockConfig(id.application())) { + try (var lock = db.lock(id.application())) { var loadBalancer = db.readLoadBalancer(id); loadBalancer.ifPresent(db::writeLoadBalancer); } @@ -73,7 +73,7 @@ public class LoadBalancerProvisioner { if (requestedNodes.type() != NodeType.tenant) return; // Nothing to provision for this node type if (!cluster.type().isContainer()) return; // Nothing to provision for this cluster type if (application.instance().isTester()) return; // Do not provision for tester instances - try (var lock = db.lockConfig(application)) { + try (var lock = db.lock(application)) { provision(application, effectiveId(cluster), false, lock); } } @@ -90,7 +90,7 @@ public class LoadBalancerProvisioner { */ public void activate(ApplicationId application, Set<ClusterSpec> clusters, @SuppressWarnings("unused") Mutex applicationLock, NestedTransaction transaction) { - try (var lock = db.lockConfig(application)) { + try (var lock = db.lock(application)) { var containerClusters = containerClustersOf(clusters); for (var clusterId : containerClusters) { // Provision again to ensure that load balancer instance is re-configured with correct nodes @@ -108,7 +108,7 @@ public class LoadBalancerProvisioner { */ public void deactivate(ApplicationId application, NestedTransaction transaction) { try (var applicationLock = nodeRepository.lock(application)) { - try (var lock = db.lockConfig(application)) { + try (var lock = db.lock(application)) { deactivate(nodeRepository.loadBalancers(application).asList(), transaction); } } @@ -166,7 +166,7 @@ public class LoadBalancerProvisioner { reals.add(new Real(HostName.from(node.hostname()), ip)); } } - log.log(LogLevel.DEBUG, "Creating load balancer for " + cluster + " in " + application.toShortString() + + log.log(Level.FINE, "Creating load balancer for " + cluster + " in " + application.toShortString() + ", targeting: " + reals); try { return service.create(application, cluster, reals, force); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 652fd5c6861..896a5f78d93 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -52,11 +52,11 @@ class NodeAllocation { /** The nodes this has accepted so far */ private final Set<PrioritizableNode> nodes = new LinkedHashSet<>(); - /** The number of nodes in the accepted nodes which are of the requested flavor */ - private int acceptedOfRequestedFlavor = 0; + /** The number of already allocated nodes accepted and not retired */ + private int accepted = 0; - /** The number of nodes in the accepted nodes which are of the requested flavor and not already retired */ - private int acceptedNonretiredOfRequestedFlavor = 0; + /** The number of already allocated nodes accepted and not retired and not needing resize */ + private int acceptedWithoutResize = 0; /** The number of nodes rejected because of clashing parentHostname */ private int rejectedWithClashingParentHost = 0; @@ -118,14 +118,14 @@ class NodeAllocation { if (offered.status().wantToRetire()) wantToRetireNode = true; if (requestedNodes.isExclusive() && ! hostsOnly(application.tenant(), offered.parentHostname())) wantToRetireNode = true; - if (( ! saturatedByNonretired() && hasCompatibleFlavor(node)) || acceptToRetire(node)) + if ((! saturated() && hasCompatibleFlavor(node)) || acceptToRetire(node)) accepted.add(acceptNode(node, wantToRetireNode, node.isResizable)); } else { accepted.add(acceptNode(node, false, false)); } } - else if ( ! saturated() && hasCompatibleFlavor(node)) { + else if (! saturated() && hasCompatibleFlavor(node)) { if ( violatesParentHostPolicy(this.nodes, offered)) { ++rejectedWithClashingParentHost; continue; @@ -226,22 +226,22 @@ class NodeAllocation { return requestedNodes.isCompatible(node.node.flavor(), flavors) || node.isResizable; } - private Node acceptNode(PrioritizableNode prioritizableNode, boolean wantToRetire, boolean resize) { + private Node acceptNode(PrioritizableNode prioritizableNode, boolean wantToRetire, boolean resizeable) { Node node = prioritizableNode.node; if (node.allocation().isPresent()) // Record the currently requested resources node = node.with(node.allocation().get().withRequestedResources(requestedNodes.resources().orElse(node.flavor().resources()))); if (! wantToRetire) { - if (resize && ! ( node.allocation().isPresent() && node.allocation().get().membership().retired())) + accepted++; + if (node.allocation().isEmpty() || ! requestedNodes.needsResize(node)) + acceptedWithoutResize++; + + if (resizeable && ! ( node.allocation().isPresent() && node.allocation().get().membership().retired())) node = resize(node); if (node.state() != Node.State.active) // reactivated node - make sure its not retired node = node.unretire(); - - acceptedOfRequestedFlavor++; - if ( ! (node.allocation().isPresent() && node.allocation().get().membership().retired())) - acceptedNonretiredOfRequestedFlavor++; } else { ++wasRetiredJustNow; // Retire nodes which are of an unwanted flavor, retired flavor or have an overlapping parent host @@ -272,29 +272,24 @@ class NodeAllocation { /** Returns true if no more nodes are needed in this list */ private boolean saturated() { - return requestedNodes.saturatedBy(acceptedOfRequestedFlavor); - } - - /** Returns true if no more nodes are needed in this list to not make changes to the retired set */ - private boolean saturatedByNonretired() { - return requestedNodes.saturatedBy(acceptedNonretiredOfRequestedFlavor); + return requestedNodes.saturatedBy(acceptedWithoutResize); } /** Returns true if the content of this list is sufficient to meet the request */ boolean fulfilled() { - return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor); + return requestedNodes.fulfilledBy(accepted); } boolean wouldBeFulfilledWithRetiredNodes() { - return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + wasRetiredJustNow); + return requestedNodes.fulfilledBy(accepted + wasRetiredJustNow); } boolean wouldBeFulfilledWithClashingParentHost() { - return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + rejectedWithClashingParentHost); + return requestedNodes.fulfilledBy(accepted + rejectedWithClashingParentHost); } boolean wouldBeFulfilledWithoutExclusivity() { - return requestedNodes.fulfilledBy(acceptedOfRequestedFlavor + rejectedDueToExclusivity); + return requestedNodes.fulfilledBy(accepted + rejectedDueToExclusivity); } /** @@ -307,7 +302,7 @@ class NodeAllocation { Optional<FlavorCount> getFulfilledDockerDeficit() { return Optional.of(requestedNodes) .filter(NodeSpec.CountNodeSpec.class::isInstance) - .map(spec -> new FlavorCount(spec.resources().get(), spec.fulfilledDeficitCount(acceptedOfRequestedFlavor))) + .map(spec -> new FlavorCount(spec.resources().get(), spec.fulfilledDeficitCount(accepted))) .filter(flavorCount -> flavorCount.getCount() > 0); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java index 524b524f8c4..c33a61b0ea3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodePrioritizer.java @@ -5,7 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -149,13 +149,13 @@ public class NodePrioritizer { if (!hostHasCapacityForWantedFlavor || conflictingCluster) continue; - log.log(LogLevel.DEBUG, "Trying to add new Docker node on " + host); + log.log(Level.FINE, "Trying to add new Docker node on " + host); Optional<IP.Allocation> allocation; try { allocation = host.ipConfig().pool().findAllocation(allNodes, nameResolver); if (allocation.isEmpty()) continue; // No free addresses in this pool } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed allocating IP address on " + host.hostname(), e); + log.log(Level.WARNING, "Failed allocating IP address on " + host.hostname(), e); continue; } @@ -167,7 +167,7 @@ public class NodePrioritizer { NodeType.tenant); PrioritizableNode nodePri = toPrioritizable(newNode, false, true); if ( ! nodePri.violatesSpares || isAllocatingForReplacement) { - log.log(LogLevel.DEBUG, "Adding new Docker node " + newNode); + log.log(Level.FINE, "Adding new Docker node " + newNode); nodes.put(newNode, nodePri); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 103543917bc..4000354243f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -14,7 +14,7 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.FlagSource; @@ -73,18 +73,6 @@ public class NodeRepositoryProvisioner implements Provisioner { this.activator = new Activator(nodeRepository, loadBalancerProvisioner); } - - /** - * Returns a list of nodes in the prepared or active state, matching the given constraints. - * The nodes are ordered by increasing index number. - */ - @Deprecated // TODO: Remove after April 2020 - @Override - public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity requestedCapacity, - int wantedGroups, ProvisionLogger logger) { - return prepare(application, cluster, requestedCapacity.withGroups(wantedGroups), logger); - } - /** * Returns a list of nodes in the prepared or active state, matching the given constraints. * The nodes are ordered by increasing index number. @@ -92,7 +80,7 @@ public class NodeRepositoryProvisioner implements Provisioner { @Override public List<HostSpec> prepare(ApplicationId application, ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { - log.log(zone.system().isCd() ? Level.INFO : LogLevel.DEBUG, + log.log(zone.system().isCd() ? Level.INFO : Level.FINE, () -> "Received deploy prepare request for " + requested + " for application " + application + ", cluster " + cluster); @@ -141,7 +129,7 @@ public class NodeRepositoryProvisioner implements Provisioner { /** * Returns the target cluster resources, a value between the min and max in the requested capacity, - * and updates the application store with the received min and max, + * and updates the application store with the received min and max. */ private ClusterResources decideTargetResources(ApplicationId applicationId, ClusterSpec.Id clusterId, Capacity requested) { try (Mutex lock = nodeRepository.lock(applicationId)) { @@ -149,32 +137,60 @@ public class NodeRepositoryProvisioner implements Provisioner { application = application.withClusterLimits(clusterId, requested.minResources(), requested.maxResources()); nodeRepository.applications().put(application, lock); return application.clusters().get(clusterId).targetResources() - .orElseGet(() -> currentResources(applicationId, clusterId, requested) - .orElse(requested.minResources())); + .orElseGet(() -> currentResources(applicationId, clusterId, requested)); } } - /** Returns the current resources of this cluster, if it's already deployed and inside the requested limits */ - private Optional<ClusterResources> currentResources(ApplicationId applicationId, - ClusterSpec.Id clusterId, - Capacity requested) { + /** Returns the current resources of this cluster, or the closes */ + private ClusterResources currentResources(ApplicationId applicationId, + ClusterSpec.Id clusterId, + Capacity requested) { List<Node> nodes = NodeList.copyOf(nodeRepository.getNodes(applicationId, Node.State.active)) .cluster(clusterId) .not().retired() .not().removable() .asList(); - if (nodes.isEmpty()) return Optional.empty(); + if (nodes.isEmpty()) return requested.minResources(); // New deployment: Start at min + long groups = nodes.stream().map(node -> node.allocation().get().membership().cluster().group()).distinct().count(); + var currentResources = new ClusterResources(nodes.size(), (int)groups, nodes.get(0).flavor().resources()); + return ensureWithin(requested.minResources(), requested.maxResources(), currentResources); + } - // To allow non-numeric settings to be updated without resetting to the min target, we need to use - // the non-numeric settings of the current min limit with the current numeric settings - NodeResources nodeResources = nodes.get(0).allocation().get().requestedResources() - .with(requested.minResources().nodeResources().diskSpeed()) - .with(requested.maxResources().nodeResources().storageType()); - var currentResources = new ClusterResources(nodes.size(), (int)groups, nodeResources); - if ( ! currentResources.isWithin(requested.minResources(), requested.maxResources())) return Optional.empty(); + /** Make the minimal adjustments needed to the current resources to stay within the limits */ + private ClusterResources ensureWithin(ClusterResources min, ClusterResources max, ClusterResources current) { + int nodes = between(min.nodes(), max.nodes(), current.nodes()); + int groups = between(min.groups(), max.groups(), current.groups()); + if (nodes % groups != 0) { + // That didn't work - try to preserve current group size instead. + // Rounding here is needed because a node may be missing due to node failing. + int currentGroupsSize = Math.round((float)current.nodes() / current.groups()); + nodes = currentGroupsSize * groups; + if (nodes != between(min.nodes(), max.nodes(), nodes)) { + // Give up: Use max + nodes = max.nodes(); + groups = max.groups(); + } + } + if (min.nodeResources() != NodeResources.unspecified && max.nodeResources() != NodeResources.unspecified) { + double vcpu = between(min.nodeResources().vcpu(), max.nodeResources().vcpu(), current.nodeResources().vcpu()); + double memoryGb = between(min.nodeResources().memoryGb(), max.nodeResources().memoryGb(), current.nodeResources().memoryGb()); + double diskGb = between(min.nodeResources().diskGb(), max.nodeResources().diskGb(), current.nodeResources().diskGb()); + // Combine computed scaled resources with requested non-scaled resources (for which min=max) + NodeResources nodeResources = min.nodeResources().withVcpu(vcpu).withMemoryGb(memoryGb).withDiskGb(diskGb); + return new ClusterResources(nodes, groups, nodeResources); + } + else { + return new ClusterResources(nodes, groups, current.nodeResources()); + } + } + + private int between(int min, int max, int n) { + return Math.min(max, Math.max(min, n)); + } - return Optional.of(currentResources); + private double between(double min, double max, double n) { + return Math.min(max, Math.max(min, n)); } private void logIfDownscaled(int targetNodes, int actualNodes, ClusterSpec cluster, ProvisionLogger logger) { @@ -195,7 +211,7 @@ public class NodeRepositoryProvisioner implements Provisioner { nodes.sort(Comparator.comparingInt(node -> node.allocation().get().membership().index())); List<HostSpec> hosts = new ArrayList<>(nodes.size()); for (Node node : nodes) { - log.log(LogLevel.DEBUG, () -> "Prepared node " + node.hostname() + " - " + node.flavor()); + log.log(Level.FINE, () -> "Prepared node " + node.hostname() + " - " + node.flavor()); Allocation nodeAllocation = node.allocation().orElseThrow(IllegalStateException::new); hosts.add(new HostSpec(node.hostname(), List.of(), @@ -206,7 +222,7 @@ public class NodeRepositoryProvisioner implements Provisioner { requestedResources == NodeResources.unspecified ? Optional.empty() : Optional.of(requestedResources), node.status().dockerImage())); if (nodeAllocation.networkPorts().isPresent()) { - log.log(LogLevel.DEBUG, () -> "Prepared node " + node.hostname() + " has port allocations"); + log.log(Level.FINE, () -> "Prepared node " + node.hostname() + " has port allocations"); } } return hosts; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index 45fc1e6934c..a8abdc3f38a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.Node; import java.util.Objects; import java.util.Optional; @@ -56,6 +57,9 @@ public interface NodeSpec { /** Returns the resources requested by this or empty if none are explicitly requested */ Optional<NodeResources> resources(); + /** Returns whether the given node must be resized to match this spec */ + boolean needsResize(Node node); + /** * Returns true if a node with given current resources and current spare host resources can be resized * in-place to resources in this spec. @@ -134,6 +138,11 @@ public interface NodeSpec { } @Override + public boolean needsResize(Node node) { + return ! node.flavor().resources().compatibleWith(requestedNodeResources); + } + + @Override public boolean canResize(NodeResources currentNodeResources, NodeResources currentSpareHostResources, boolean hasTopologyChange, int currentClusterSize) { // Never allow in-place resize when also changing topology or decreasing cluster size @@ -199,6 +208,9 @@ public interface NodeSpec { public Optional<NodeResources> resources() { return Optional.empty(); } @Override + public boolean needsResize(Node node) { return false; } + + @Override public String toString() { return "request for all nodes of type '" + type + "'"; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java new file mode 100644 index 00000000000..90081ecb604 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ApplicationSerializer.java @@ -0,0 +1,47 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Cluster; + +import java.net.URI; +import java.util.Collection; + +/** + * Serializes application information for nodes/v2/application responses + */ +public class ApplicationSerializer { + + public static Slime toSlime(Application application, URI applicationUri) { + Slime slime = new Slime(); + toSlime(application, slime.setObject(), applicationUri); + return slime; + } + + private static void toSlime(Application application, Cursor object, URI applicationUri) { + object.setString("url", applicationUri.toString()); + object.setString("id", application.id().toFullString()); + clustersToSlime(application.clusters().values(), object.setObject("clusters")); + } + + private static void clustersToSlime(Collection<Cluster> clusters, Cursor clustersObject) { + clusters.forEach(cluster -> toSlime(cluster, clustersObject.setObject(cluster.id().value()))); + } + + private static void toSlime(Cluster cluster, Cursor clusterObject) { + toSlime(cluster.minResources(), clusterObject.setObject("min")); + toSlime(cluster.maxResources(), clusterObject.setObject("max")); + cluster.suggestedResources().ifPresent(suggested -> toSlime(suggested, clusterObject.setObject("suggested"))); + cluster.targetResources().ifPresent(target -> toSlime(target, clusterObject.setObject("target"))); + } + + private static void toSlime(ClusterResources resources, Cursor clusterResourcesObject) { + clusterResourcesObject.setLong("nodes", resources.nodes()); + clusterResourcesObject.setLong("groups", resources.groups()); + NodeResourcesSerializer.toSlime(resources.nodeResources(), clusterResourcesObject.setObject("resources")); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java index 488b11e627b..12a29707303 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/HostCapacityResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/HostCapacityResponse.java @@ -1,5 +1,5 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java index 4dfdef742d6..3b6de41f142 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/JobsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/JobsResponse.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index 3147d4caded..092426c75d4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -1,5 +1,5 @@ -// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java index 24aa1fe0b25..f81e3240397 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/LoadBalancersApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiHandler.java @@ -1,5 +1,5 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -15,12 +15,12 @@ import java.util.logging.Level; /** * @author mpolden */ -public class LoadBalancersApiHandler extends LoggingRequestHandler { +public class LoadBalancersV1ApiHandler extends LoggingRequestHandler { private final NodeRepository nodeRepository; @Inject - public LoadBalancersApiHandler(LoggingRequestHandler.Context parentCtx, NodeRepository nodeRepository) { + public LoadBalancersV1ApiHandler(LoggingRequestHandler.Context parentCtx, NodeRepository nodeRepository) { super(parentCtx); this.nodeRepository = nodeRepository; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java index 9947d2d5108..a7577392fe2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeAclResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeAclResponse.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 12b245ede7f..8b5639cc514 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -1,5 +1,5 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.component.Version; import com.yahoo.config.provision.DockerImage; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java new file mode 100644 index 00000000000..625ee89434e --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeResourcesSerializer.java @@ -0,0 +1,57 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.config.provision.NodeResources; +import com.yahoo.slime.Cursor; + +/** + * @author bratseth + */ +public class NodeResourcesSerializer { + + static void toSlime(NodeResources resources, Cursor object) { + object.setDouble("vcpu", resources.vcpu()); + object.setDouble("memoryGb", resources.memoryGb()); + object.setDouble("diskGb", resources.diskGb()); + object.setDouble("bandwidthGbps", resources.bandwidthGbps()); + object.setString("diskSpeed", toString(resources.diskSpeed())); + object.setString("storageType", toString(resources.storageType())); + } + + public static NodeResources.DiskSpeed diskSpeedFrom(String diskSpeed) { + switch (diskSpeed) { + case "fast": return NodeResources.DiskSpeed.fast; + case "slow": return NodeResources.DiskSpeed.slow; + case "any" : return NodeResources.DiskSpeed.any; + default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed + "'"); + } + } + + private static String toString(NodeResources.DiskSpeed diskSpeed) { + switch (diskSpeed) { + case fast : return "fast"; + case slow : return "slow"; + case any : return "any"; + default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed.name() + "'"); + } + } + + public static NodeResources.StorageType storageTypeFrom(String storageType) { + switch (storageType) { + case "local" : return NodeResources.StorageType.local; + case "remote": return NodeResources.StorageType.remote; + case "any" : return NodeResources.StorageType.any; + default: throw new IllegalArgumentException("Unknown storage type '" + storageType + "'"); + } + } + + private static String toString(NodeResources.StorageType storageType) { + switch (storageType) { + case remote : return "remote"; + case local : return "local"; + case any : return "any"; + default: throw new IllegalArgumentException("Unknown storage type '" + storageType.name() + "'"); + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java index 492994c2f0e..bd65894101c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializer.java @@ -1,8 +1,9 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.slime.Cursor; import com.yahoo.vespa.hosted.provision.Node; /** @@ -12,7 +13,7 @@ import com.yahoo.vespa.hosted.provision.Node; */ public class NodeSerializer { - public Node.State stateFrom(String state) { + public static Node.State stateFrom(String state) { switch (state) { case "active": return Node.State.active; case "dirty": return Node.State.dirty; @@ -27,7 +28,7 @@ public class NodeSerializer { } } - public String toString(Node.State state) { + public static String toString(Node.State state) { switch (state) { case active: return "active"; case dirty: return "dirty"; @@ -42,7 +43,7 @@ public class NodeSerializer { } } - public NodeType typeFrom(String nodeType) { + public static NodeType typeFrom(String nodeType) { switch (nodeType) { case "tenant": return NodeType.tenant; case "host": return NodeType.host; @@ -57,7 +58,7 @@ public class NodeSerializer { } } - public String toString(NodeType type) { + public static String toString(NodeType type) { switch (type) { case tenant: return "tenant"; case host: return "host"; @@ -72,40 +73,4 @@ public class NodeSerializer { } } - public NodeResources.DiskSpeed diskSpeedFrom(String diskSpeed) { - switch (diskSpeed) { - case "fast": return NodeResources.DiskSpeed.fast; - case "slow": return NodeResources.DiskSpeed.slow; - case "any" : return NodeResources.DiskSpeed.any; - default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed + "'"); - } - } - - public String toString(NodeResources.DiskSpeed diskSpeed) { - switch (diskSpeed) { - case fast : return "fast"; - case slow : return "slow"; - case any : return "any"; - default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed.name() + "'"); - } - } - - public NodeResources.StorageType storageTypeFrom(String storageType) { - switch (storageType) { - case "local" : return NodeResources.StorageType.local; - case "remote": return NodeResources.StorageType.remote; - case "any" : return NodeResources.StorageType.any; - default: throw new IllegalArgumentException("Unknown storage type '" + storageType + "'"); - } - } - - public String toString(NodeResources.StorageType storageType) { - switch (storageType) { - case remote : return "remote"; - case local : return "local"; - case any : return "any"; - default: throw new IllegalArgumentException("Unknown storage type '" + storageType.name() + "'"); - } - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index a4412a502aa..9d02d2907cc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -1,11 +1,10 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.container.jdisc.HttpRequest; @@ -48,14 +47,13 @@ class NodesResponse extends HttpResponse { private final Function<HostName, Optional<HostInfo>> orchestrator; private final NodeRepository nodeRepository; private final Slime slime; - private final NodeSerializer serializer = new NodeSerializer(); public NodesResponse(ResponseType responseType, HttpRequest request, Orchestrator orchestrator, NodeRepository nodeRepository) { super(200); this.parentUrl = toParentUrl(request); this.nodeParentUrl = toNodeParentUrl(request); - filter = NodesApiHandler.toNodeFilter(request); + filter = NodesV2ApiHandler.toNodeFilter(request); this.recursive = request.getBooleanProperty("recursive"); this.orchestrator = orchestrator.getHostResolver(); this.nodeRepository = nodeRepository; @@ -65,7 +63,7 @@ class NodesResponse extends HttpResponse { switch (responseType) { case nodeList: nodesToSlime(root); break; case stateList : statesToSlime(root); break; - case nodesInStateList: nodesToSlime(serializer.stateFrom(lastElement(parentUrl)), root); break; + case nodesInStateList: nodesToSlime(NodeSerializer.stateFrom(lastElement(parentUrl)), root); break; case singleNode : nodeToSlime(lastElement(parentUrl), root); break; default: throw new IllegalArgumentException(); } @@ -97,11 +95,11 @@ class NodesResponse extends HttpResponse { private void statesToSlime(Cursor root) { Cursor states = root.setObject("states"); for (Node.State state : Node.State.values()) - toSlime(state, states.setObject(serializer.toString(state))); + toSlime(state, states.setObject(NodeSerializer.toString(state))); } private void toSlime(Node.State state, Cursor object) { - object.setString("url", parentUrl + serializer.toString(state)); + object.setString("url", parentUrl + NodeSerializer.toString(state)); if (recursive) nodesToSlime(state, object); } @@ -136,10 +134,10 @@ class NodesResponse extends HttpResponse { object.setString("url", nodeParentUrl + node.hostname()); if ( ! allFields) return; object.setString("id", node.hostname()); - object.setString("state", serializer.toString(node.state())); + object.setString("state", NodeSerializer.toString(node.state())); object.setString("type", node.type().name()); object.setString("hostname", node.hostname()); - object.setString("type", serializer.toString(node.type())); + object.setString("type", NodeSerializer.toString(node.type())); if (node.parentHostname().isPresent()) { object.setString("parentHostname", node.parentHostname().get()); } @@ -148,7 +146,7 @@ class NodesResponse extends HttpResponse { node.reservedTo().ifPresent(reservedTo -> object.setString("reservedTo", reservedTo.value())); if (node.flavor().isConfigured()) object.setDouble("cpuCores", node.flavor().getMinCpuCores()); - toSlime(node.flavor().resources(), object.setObject("resources")); + NodeResourcesSerializer.toSlime(node.flavor().resources(), object.setObject("resources")); if (node.flavor().cost() > 0) object.setLong("cost", node.flavor().cost()); object.setString("environment", node.flavor().getType().name()); @@ -160,7 +158,7 @@ class NodesResponse extends HttpResponse { object.setString("wantedDockerImage", allocation.membership().cluster().dockerImage() .orElseGet(() -> nodeRepository.dockerImage(node).withTag(allocation.membership().cluster().vespaVersion()).asString())); object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString()); - toSlime(allocation.requestedResources(), object.setObject("requestedResources")); + NodeResourcesSerializer.toSlime(allocation.requestedResources(), object.setObject("requestedResources")); allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts"))); orchestrator.apply(new HostName(node.hostname())) .ifPresent(info -> { @@ -210,15 +208,6 @@ class NodesResponse extends HttpResponse { } } - private void toSlime(NodeResources resources, Cursor object) { - object.setDouble("vcpu", resources.vcpu()); - object.setDouble("memoryGb", resources.memoryGb()); - object.setDouble("diskGb", resources.diskGb()); - object.setDouble("bandwidthGbps", resources.bandwidthGbps()); - object.setString("diskSpeed", serializer.toString(resources.diskSpeed())); - object.setString("storageType", serializer.toString(resources.storageType())); - } - // Hack: For non-docker nodes, return current docker image as default prefix + current Vespa version // TODO: Remove current + wanted docker image from response for non-docker types private Optional<DockerImage> currentDockerImage(Node node) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 809e4200e7e..75b1efb649e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodesApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -1,7 +1,8 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; @@ -17,14 +18,18 @@ import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.MessageResponse; import com.yahoo.restapi.Path; import com.yahoo.restapi.ResourceResponse; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.NoSuchNodeException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; @@ -34,7 +39,7 @@ import com.yahoo.vespa.hosted.provision.node.filter.NodeOsVersionFilter; import com.yahoo.vespa.hosted.provision.node.filter.NodeTypeFilter; import com.yahoo.vespa.hosted.provision.node.filter.ParentHostFilter; import com.yahoo.vespa.hosted.provision.node.filter.StateFilter; -import com.yahoo.vespa.hosted.provision.restapi.v2.NodesResponse.ResponseType; +import com.yahoo.vespa.hosted.provision.restapi.NodesResponse.ResponseType; import com.yahoo.vespa.orchestrator.Orchestrator; import com.yahoo.yolean.Exceptions; @@ -42,6 +47,8 @@ import javax.inject.Inject; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -59,16 +66,15 @@ import static com.yahoo.slime.SlimeUtils.optionalString; * * @author bratseth */ -public class NodesApiHandler extends LoggingRequestHandler { +public class NodesV2ApiHandler extends LoggingRequestHandler { private final Orchestrator orchestrator; private final NodeRepository nodeRepository; private final NodeFlavors nodeFlavors; - private final NodeSerializer serializer = new NodeSerializer(); @Inject - public NodesApiHandler(LoggingRequestHandler.Context parentCtx, Orchestrator orchestrator, - NodeRepository nodeRepository, NodeFlavors flavors) { + public NodesV2ApiHandler(LoggingRequestHandler.Context parentCtx, Orchestrator orchestrator, + NodeRepository nodeRepository, NodeFlavors flavors) { super(parentCtx); this.orchestrator = orchestrator; this.nodeRepository = nodeRepository; @@ -100,18 +106,21 @@ public class NodesApiHandler extends LoggingRequestHandler { } private HttpResponse handleGET(HttpRequest request) { - String path = request.getUri().getPath(); - if (path.equals( "/nodes/v2/")) return new ResourceResponse(request.getUri(), "state", "node", "command", "maintenance", "upgrade"); - if (path.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, orchestrator, nodeRepository); - if (path.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, orchestrator, nodeRepository); - if (path.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, orchestrator, nodeRepository); - if (path.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, orchestrator, nodeRepository); - if (path.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); - if (path.equals( "/nodes/v2/command/")) return new ResourceResponse(request.getUri(), "restart", "reboot"); - if (path.equals( "/nodes/v2/maintenance/")) return new JobsResponse(nodeRepository.jobControl()); - if (path.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(nodeRepository.infrastructureVersions(), nodeRepository.osVersions(), nodeRepository.dockerImages()); - if (path.startsWith("/nodes/v2/capacity")) return new HostCapacityResponse(nodeRepository, request); - throw new NotFoundException("Nothing at path '" + path + "'"); + Path path = new Path(request.getUri()); + String pathS = request.getUri().getPath(); + if (pathS.equals( "/nodes/v2/")) return new ResourceResponse(request.getUri(), "state", "node", "command", "maintenance", "upgrade", "application"); + if (pathS.equals( "/nodes/v2/node/")) return new NodesResponse(ResponseType.nodeList, request, orchestrator, nodeRepository); + if (pathS.startsWith("/nodes/v2/node/")) return new NodesResponse(ResponseType.singleNode, request, orchestrator, nodeRepository); + if (pathS.equals( "/nodes/v2/state/")) return new NodesResponse(ResponseType.stateList, request, orchestrator, nodeRepository); + if (pathS.startsWith("/nodes/v2/state/")) return new NodesResponse(ResponseType.nodesInStateList, request, orchestrator, nodeRepository); + if (pathS.startsWith("/nodes/v2/acl/")) return new NodeAclResponse(request, nodeRepository); + if (pathS.equals( "/nodes/v2/command/")) return new ResourceResponse(request.getUri(), "restart", "reboot"); + if (pathS.equals( "/nodes/v2/maintenance/")) return new JobsResponse(nodeRepository.jobControl()); + if (pathS.equals( "/nodes/v2/upgrade/")) return new UpgradeResponse(nodeRepository.infrastructureVersions(), nodeRepository.osVersions(), nodeRepository.dockerImages()); + if (pathS.startsWith("/nodes/v2/capacity")) return new HostCapacityResponse(nodeRepository, request); + if (path.matches("/nodes/v2/application")) return applicationList(request.getUri()); + if (path.matches("/nodes/v2/application/{applicationId}")) return application(path.get("applicationId"), request.getUri()); + throw new NotFoundException("Nothing at " + path); } private HttpResponse handlePUT(HttpRequest request) { @@ -189,17 +198,29 @@ public class NodesApiHandler extends LoggingRequestHandler { private HttpResponse handleDELETE(HttpRequest request) { Path path = new Path(request.getUri()); - if (path.matches("/nodes/v2/node/{hostname}")) { - String hostname = path.get("hostname"); - List<Node> removedNodes = nodeRepository.removeRecursively(hostname); - return new MessageResponse("Removed " + removedNodes.stream().map(Node::hostname).collect(Collectors.joining(", "))); - } + if (path.matches("/nodes/v2/node/{hostname}")) return deleteNode(path.get("hostname")); if (path.matches("/nodes/v2/maintenance/inactive/{job}")) return setJobActive(path.get("job"), true); if (path.matches("/nodes/v2/upgrade/firmware")) return cancelFirmwareCheckResponse(); throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); } + private HttpResponse deleteNode(String hostname) { + Optional<Node> node = nodeRepository.getNode(hostname); + if (node.isEmpty()) throw new NotFoundException("No node with hostname '" + hostname + "'"); + try (Mutex lock = nodeRepository.lock(node.get())) { + node = nodeRepository.getNode(hostname); + if (node.isEmpty()) throw new NotFoundException("No node with hostname '" + hostname + "'"); + if (node.get().state() == Node.State.deprovisioned) { + nodeRepository.forget(node.get()); + return new MessageResponse("Permanently removed " + hostname); + } else { + List<Node> removedNodes = nodeRepository.removeRecursively(hostname); + return new MessageResponse("Removed " + removedNodes.stream().map(Node::hostname).collect(Collectors.joining(", "))); + } + } + } + private Node nodeFromRequest(HttpRequest request) { String hostname = lastElement(request.getUri().getPath()); return nodeRepository.getNode(hostname).orElseThrow(() -> @@ -254,8 +275,8 @@ public class NodesApiHandler extends LoggingRequestHandler { requiredField(resourcesInspector, "memoryGb", Inspector::asDouble), requiredField(resourcesInspector, "diskGb", Inspector::asDouble), requiredField(resourcesInspector, "bandwidthGbps", Inspector::asDouble), - optionalString(resourcesInspector.field("diskSpeed")).map(serializer::diskSpeedFrom).orElse(NodeResources.DiskSpeed.getDefault()), - optionalString(resourcesInspector.field("storageType")).map(serializer::storageTypeFrom).orElse(NodeResources.StorageType.getDefault()))); + optionalString(resourcesInspector.field("diskSpeed")).map(NodeResourcesSerializer::diskSpeedFrom).orElse(NodeResources.DiskSpeed.getDefault()), + optionalString(resourcesInspector.field("storageType")).map(NodeResourcesSerializer::storageTypeFrom).orElse(NodeResources.StorageType.getDefault()))); } Flavor flavor = nodeFlavors.getFlavorOrThrow(flavorInspector.asString()); @@ -269,9 +290,9 @@ public class NodesApiHandler extends LoggingRequestHandler { if (resourcesInspector.field("bandwidthGbps").valid()) flavor = flavor.with(flavor.resources().withBandwidthGbps(resourcesInspector.field("bandwidthGbps").asDouble())); if (resourcesInspector.field("diskSpeed").valid()) - flavor = flavor.with(flavor.resources().with(serializer.diskSpeedFrom(resourcesInspector.field("diskSpeed").asString()))); + flavor = flavor.with(flavor.resources().with(NodeResourcesSerializer.diskSpeedFrom(resourcesInspector.field("diskSpeed").asString()))); if (resourcesInspector.field("storageType").valid()) - flavor = flavor.with(flavor.resources().with(serializer.storageTypeFrom(resourcesInspector.field("storageType").asString()))); + flavor = flavor.with(flavor.resources().with(NodeResourcesSerializer.storageTypeFrom(resourcesInspector.field("storageType").asString()))); } return flavor; } @@ -284,7 +305,7 @@ public class NodesApiHandler extends LoggingRequestHandler { private NodeType nodeTypeFromSlime(Inspector object) { if (! object.valid()) return NodeType.tenant; // default - return serializer.typeFrom(object.asString()); + return NodeSerializer.typeFrom(object.asString()); } private Optional<TenantName> reservedToFromSlime(Inspector object) { @@ -394,4 +415,35 @@ public class NodesApiHandler extends LoggingRequestHandler { return nodes.stream().map(Node::hostname).sorted().collect(Collectors.joining(", ")); } + private HttpResponse applicationList(URI uri) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Cursor applications = root.setArray("applications"); + for (ApplicationId id : nodeRepository.applications().ids()) { + Cursor application = applications.addObject(); + application.setString("url", withPath("/nodes/v2/applications/" + id.toFullString(), uri).toString()); + application.setString("id", id.toFullString()); + } + return new SlimeJsonResponse(slime); + } + + private HttpResponse application(String id, URI uri) { + Optional<Application> application = nodeRepository.applications().get(ApplicationId.fromFullString(id)); + if (application.isEmpty()) + return ErrorResponse.notFoundError("No application '" + id + "'"); + Slime slime = ApplicationSerializer.toSlime(application.get(), + withPath("/nodes/v2/applications/" + id, uri)); + return new SlimeJsonResponse(slime); + } + + /** Returns a copy of the given URI with the host and port from the given URI and the path set to the given path */ + private URI withPath(String newPath, URI uri) { + try { + return new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), uri.getPort(), newPath, null, null); + } + catch (URISyntaxException e) { + throw new RuntimeException("Will not happen", e); + } + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NotFoundException.java index a2bfba95b8a..664aad939f2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/NotFoundException.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NotFoundException.java @@ -1,5 +1,5 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; /** * Thrown when a resource is not found diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java index 381a1bc27aa..16858ec6963 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/v2/UpgradeResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/UpgradeResponse.java @@ -1,5 +1,5 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index d26accd7a84..d66bb7d6efb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -32,10 +32,10 @@ public class ContainerConfig { " <component id='com.yahoo.vespa.hosted.provision.maintenance.NodeRepositoryMaintenance'/>\n" + " <component id='com.yahoo.vespa.flags.InMemoryFlagSource'/>\n" + " <component id='com.yahoo.config.provision.Zone'/>\n" + - " <handler id='com.yahoo.vespa.hosted.provision.restapi.v2.NodesApiHandler'>\n" + + " <handler id='com.yahoo.vespa.hosted.provision.restapi.NodesV2ApiHandler'>\n" + " <binding>http://*/nodes/v2/*</binding>\n" + " </handler>\n" + - " <handler id='com.yahoo.vespa.hosted.provision.restapi.v2.LoadBalancersApiHandler'>\n" + + " <handler id='com.yahoo.vespa.hosted.provision.restapi.LoadBalancersV1ApiHandler'>\n" + " <binding>http://*/loadbalancers/v1/*</binding>\n" + " </handler>\n" + " <http>\n" + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 8ff17a8e5a3..6fafd496174 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -16,11 +16,14 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; +import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.Status; @@ -142,9 +145,22 @@ public class MockNodeRepository extends NodeRepository { ClusterSpec zoneCluster = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("node-admin")).vespaVersion("6.42").build(); activate(provisioner.prepare(zoneApp, zoneCluster, Capacity.fromRequiredNodeType(NodeType.host), null), zoneApp, provisioner); - ApplicationId app1 = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1")); - ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1")).vespaVersion("6.42").build(); - provisioner.prepare(app1, cluster1, Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1))), null); + ApplicationId app1Id = ApplicationId.from(TenantName.from("tenant1"), ApplicationName.from("application1"), InstanceName.from("instance1")); + ClusterSpec cluster1Id = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("id1")).vespaVersion("6.42").build(); + provisioner.prepare(app1Id, + cluster1Id, + Capacity.from(new ClusterResources(2, 1, new NodeResources(2, 8, 50, 1)), + new ClusterResources(8, 2, new NodeResources(4, 16, 1000, 1)), false, true), + null); + Application app1 = applications().get(app1Id).get(); + Cluster cluster1 = app1.cluster(cluster1Id.id()).get(); + cluster1 = cluster1.withSuggested(Optional.of(new ClusterResources(6, 2, + new NodeResources(3, 20, 100, 1)))); + cluster1 = cluster1.withTarget(Optional.of(new ClusterResources(4, 1, + new NodeResources(3, 16, 100, 1)))); + try (Mutex lock = lock(app1Id)) { + applications().put(app1.with(cluster1), lock); + } ApplicationId app2 = ApplicationId.from(TenantName.from("tenant2"), ApplicationName.from("application2"), InstanceName.from("instance2")); ClusterSpec cluster2 = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("id2")).vespaVersion("6.42").build(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java index 045d0cad1ad..12731e30f46 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockProvisioner.java @@ -20,12 +20,6 @@ import java.util.List; public class MockProvisioner implements Provisioner { @Override - @Deprecated // TODO: Remove after April 2020 - public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, int groups, ProvisionLogger logger) { - return Collections.emptyList(); - } - - @Override public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { return Collections.emptyList(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md index bbb2ec7080b..f7421bf07c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/README.md @@ -1,5 +1,5 @@ <!-- Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -The test resources are used by both NoadAdmin and NodeRepository +The test resources are used by both NodeAdmin and NodeRepository tests to verify APIs. So when modifying this test data remember to check tests for both NodeAdmin and NodeRepository. diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java index 7c6c4e2336b..b4f6adc2d26 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java @@ -21,6 +21,9 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +/** + * @author bratseth + */ public class AutoscalingIntegrationTest { @Test @@ -55,8 +58,8 @@ public class AutoscalingIntegrationTest { Application application = tester.nodeRepository().applications().get(application1).orElse(new Application(application1)) .withClusterLimits(cluster1.id(), min, max); tester.nodeRepository().applications().put(application, tester.nodeRepository().lock(application1)); - var scaledResources = autoscaler.autoscale(application.clusters().get(cluster1.id()), - tester.nodeRepository().getNodes(application1)); + var scaledResources = autoscaler.suggest(application.clusters().get(cluster1.id()), + tester.nodeRepository().getNodes(application1)); assertTrue(scaledResources.isPresent()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 497a2a31ce5..f98d55511ec 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -50,9 +50,9 @@ public class AutoscalingTest { assertTrue("Too few measurements -> No change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); tester.addMeasurements(Resource.cpu, 0.25f, 1f, 60, application1); - AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", - 15, 1, 1.3, 28.6, 28.6, - tester.autoscale(application1, cluster1.id(), min, max)); + ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", + 15, 1, 1.3, 28.6, 28.6, + tester.autoscale(application1, cluster1.id(), min, max)); tester.deploy(application1, cluster1, scaledResources); assertTrue("Cluster in flux -> No further change", tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); @@ -90,11 +90,11 @@ public class AutoscalingTest { new NodeResources(1, 1, 1, 1, NodeResources.DiskSpeed.any)); ClusterResources max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)); - AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", - 15, 1, 1.3, 28.6, 28.6, - tester.autoscale(application1, cluster1.id(), min, max)); + ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high", + 15, 1, 1.3, 28.6, 28.6, + tester.autoscale(application1, cluster1.id(), min, max)); assertEquals("Disk speed from min/max is used", - NodeResources.DiskSpeed.any, scaledResources.realResources().diskSpeed()); + NodeResources.DiskSpeed.any, scaledResources.nodeResources().diskSpeed()); tester.deploy(application1, cluster1, scaledResources); tester.nodeRepository().getNodes(application1).stream() .allMatch(n -> n.allocation().get().requestedResources().diskSpeed() == NodeResources.DiskSpeed.any); @@ -115,9 +115,9 @@ public class AutoscalingTest { tester.deploy(application1, cluster1, 5, 1, resources); tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); - AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since cpu usage is too high", - 7, 1, 2.6, 80.0, 80.0, - tester.autoscale(application1, cluster1.id(), min, max)); + ClusterResources scaledResources = tester.assertResources("Scaling up since cpu usage is too high", + 7, 1, 2.6, 80.0, 80.0, + tester.autoscale(application1, cluster1.id(), min, max)); tester.deploy(application1, cluster1, scaledResources); tester.deactivateRetired(application1, cluster1, scaledResources); @@ -187,9 +187,24 @@ public class AutoscalingTest { tester.autoscale(application1, cluster1.id(), min, max)); } - /** This condition ensures we get recommendation suggestions when deactivated */ @Test - public void testAutoscalingLimitsAreIgnoredIfMinEqualsMax() { + public void testAutoscalingLimitsWhenMinEqualsMax() { + NodeResources resources = new NodeResources(3, 100, 100, 1); + ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); + ClusterResources max = min; + AutoscalingTester tester = new AutoscalingTester(resources); + + ApplicationId application1 = tester.applicationId("application1"); + ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); + + // deploy + tester.deploy(application1, cluster1, 5, 1, resources); + tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); + assertTrue(tester.autoscale(application1, cluster1.id(), min, max).isEmpty()); + } + + @Test + public void testSuggestionsIgnoresLimits() { NodeResources resources = new NodeResources(3, 100, 100, 1); ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); ClusterResources max = min; @@ -203,7 +218,7 @@ public class AutoscalingTest { tester.addMeasurements(Resource.cpu, 0.25f, 1f, 120, application1); tester.assertResources("Scaling up since resource usage is too high", 7, 1, 2.6, 80.0, 80.0, - tester.autoscale(application1, cluster1.id(), min, max)); + tester.suggest(application1, cluster1.id(), min, max)); } @Test @@ -280,7 +295,7 @@ public class AutoscalingTest { tester.deploy(application1, cluster1, 5, 1, new NodeResources(3, 103, 100, 1)); tester.addMeasurements(Resource.memory, 0.9f, 0.6f, 120, application1); - AllocatableClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high.", + ClusterResources scaledResources = tester.assertResources("Scaling up since resource usage is too high.", 8, 1, 3, 83, 34.3, tester.autoscale(application1, cluster1.id(), min, max)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index c250a0a23b0..49bb51c1d79 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -79,8 +79,8 @@ class AutoscalingTester { return ClusterSpec.request(type, ClusterSpec.Id.from(clusterId)).vespaVersion("7").build(); } - public void deploy(ApplicationId application, ClusterSpec cluster, AllocatableClusterResources resources) { - deploy(application, cluster, resources.nodes(), resources.groups(), resources.advertisedResources()); + public void deploy(ApplicationId application, ClusterSpec cluster, ClusterResources resources) { + deploy(application, cluster, resources.nodes(), resources.groups(), resources.nodeResources()); } public List<HostSpec> deploy(ApplicationId application, ClusterSpec cluster, int nodes, int groups, NodeResources resources) { @@ -101,7 +101,7 @@ class AutoscalingTester { nodeRepository().setReady(List.of(host), Agent.system, getClass().getSimpleName()); } - public void deactivateRetired(ApplicationId application, ClusterSpec cluster, AllocatableClusterResources resources) { + public void deactivateRetired(ApplicationId application, ClusterSpec cluster, ClusterResources resources) { try (Mutex lock = nodeRepository().lock(application)){ for (Node node : nodeRepository().getNodes(application, Node.State.active)) { if (node.allocation().get().membership().retired()) @@ -154,7 +154,7 @@ class AutoscalingTester { } } - public Optional<AllocatableClusterResources> autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId, + public Optional<ClusterResources> autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId, ClusterResources min, ClusterResources max) { Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)) .withClusterLimits(clusterId, min, max); @@ -163,18 +163,27 @@ class AutoscalingTester { nodeRepository().getNodes(applicationId, Node.State.active)); } - public AllocatableClusterResources assertResources(String message, + public Optional<ClusterResources> suggest(ApplicationId applicationId, ClusterSpec.Id clusterId, + ClusterResources min, ClusterResources max) { + Application application = nodeRepository().applications().get(applicationId).orElse(new Application(applicationId)) + .withClusterLimits(clusterId, min, max); + nodeRepository().applications().put(application, nodeRepository().lock(applicationId)); + return autoscaler.suggest(application.clusters().get(clusterId), + nodeRepository().getNodes(applicationId, Node.State.active)); + } + + public ClusterResources assertResources(String message, int nodeCount, int groupCount, double approxCpu, double approxMemory, double approxDisk, - Optional<AllocatableClusterResources> actualResources) { + Optional<ClusterResources> resources) { double delta = 0.0000000001; - assertTrue(message, actualResources.isPresent()); - assertEquals("Node count: " + message, nodeCount, actualResources.get().nodes()); - assertEquals("Group count: " + message, groupCount, actualResources.get().groups()); - assertEquals("Cpu: " + message, approxCpu, Math.round(actualResources.get().advertisedResources().vcpu() * 10) / 10.0, delta); - assertEquals("Memory: " + message, approxMemory, Math.round(actualResources.get().advertisedResources().memoryGb() * 10) / 10.0, delta); - assertEquals("Disk: " + message, approxDisk, Math.round(actualResources.get().advertisedResources().diskGb() * 10) / 10.0, delta); - return actualResources.get(); + assertTrue(message, resources.isPresent()); + assertEquals("Node count: " + message, nodeCount, resources.get().nodes()); + assertEquals("Group count: " + message, groupCount, resources.get().groups()); + assertEquals("Cpu: " + message, approxCpu, Math.round(resources.get().nodeResources().vcpu() * 10) / 10.0, delta); + assertEquals("Memory: " + message, approxMemory, Math.round(resources.get().nodeResources().memoryGb() * 10) / 10.0, delta); + assertEquals("Disk: " + message, approxDisk, Math.round(resources.get().nodeResources().diskGb() * 10) / 10.0, delta); + return resources.get(); } public ManualClock clock() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java index e278cdc992a..a5e96369591 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirerTest.java @@ -155,7 +155,7 @@ public class LoadBalancerExpirerTest { } private void deployApplication(ApplicationId application, boolean activate, ClusterSpec.Id... clusters) { - tester.makeReadyNodes(10, "d-1-4-10"); + tester.makeReadyNodes(10, new NodeResources(1, 4, 10, 0.3)); List<HostSpec> hosts = new ArrayList<>(); for (var cluster : clusters) { hosts.addAll(tester.prepare(application, ClusterSpec.request(ClusterSpec.Type.container, cluster).vespaVersion(Vtag.currentVersion).build(), diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java new file mode 100644 index 00000000000..40892d80759 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -0,0 +1,101 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetrics; +import com.yahoo.vespa.hosted.provision.autoscale.NodeMetricsDb; +import com.yahoo.vespa.hosted.provision.autoscale.Resource; +import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Tests the scaling suggestions maintainer integration. + * The specific suggestions are not tested here. + * + * @author bratseth + */ +public class ScalingSuggestionsMaintainerTest { + + @Test + public void testScalingSuggestionsMaintainer() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east3"))).flavorsConfig(flavorsConfig()).build(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = tester.clusterSpec(); + + ApplicationId app2 = tester.makeApplicationId("app2"); + ClusterSpec cluster2 = tester.clusterSpec(); + + NodeMetricsDb nodeMetricsDb = new NodeMetricsDb(); + + tester.makeReadyNodes(20, "flt", NodeType.host, 8); + tester.deployZoneApp(); + + tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), + new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), + false, true)); + tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), + new ClusterResources(10, 1, new NodeResources(6.5, 5, 15, 0.1)), + false, true)); + + addMeasurements(Resource.cpu, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.memory, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.disk, 0.9f, 500, app1, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.cpu, 0.99f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.memory, 0.99f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + addMeasurements(Resource.disk, 0.99f, 500, app2, tester.nodeRepository(), nodeMetricsDb); + + ScalingSuggestionsMaintainer maintainer = new ScalingSuggestionsMaintainer(tester.nodeRepository(), + tester.identityHostResourcesCalculator(), + nodeMetricsDb, + Duration.ofMinutes(1)); + maintainer.maintain(); + + assertEquals("7 nodes with [vcpu: 15.3, memory: 5.1 Gb, disk 15.0 Gb, bandwidth: 0.1 Gbps]", + tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggestedResources().get().toString()); + assertEquals("7 nodes with [vcpu: 16.8, memory: 5.7 Gb, disk 16.5 Gb, bandwidth: 0.1 Gbps]", + tester.nodeRepository().applications().get(app2).get().cluster(cluster2.id()).get().suggestedResources().get().toString()); + } + + public void addMeasurements(Resource resource, float value, int count, ApplicationId applicationId, + NodeRepository nodeRepository, NodeMetricsDb db) { + List<Node> nodes = nodeRepository.getNodes(applicationId, Node.State.active); + for (int i = 0; i < count; i++) { + for (Node node : nodes) + db.add(List.of(new NodeMetrics.MetricValue(node.hostname(), + resource.metricName(), + nodeRepository.clock().instant().toEpochMilli(), + value * 100))); // the metrics are in % + } + } + + private FlavorsConfig flavorsConfig() { + FlavorConfigBuilder b = new FlavorConfigBuilder(); + b.addFlavor("flt", 30, 30, 40, 3, Flavor.Type.BARE_METAL); + b.addFlavor("cpu", 40, 20, 40, 3, Flavor.Type.BARE_METAL); + b.addFlavor("mem", 20, 40, 40, 3, Flavor.Type.BARE_METAL); + return b.build(); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index 1f07e26d045..c50805eebb8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -28,10 +28,12 @@ public class ApplicationSerializerTest { clusters.add(new Cluster(ClusterSpec.Id.from("c1"), new ClusterResources( 8, 4, new NodeResources(1, 2, 3, 4)), new ClusterResources(12, 6, new NodeResources(3, 6, 21, 24)), + Optional.empty(), Optional.empty())); clusters.add(new Cluster(ClusterSpec.Id.from("c2"), new ClusterResources( 8, 4, new NodeResources(1, 2, 3, 4)), new ClusterResources(14, 7, new NodeResources(3, 6, 21, 24)), + Optional.of(new ClusterResources(20, 10, new NodeResources(0.5, 4, 14, 16))), Optional.of(new ClusterResources(10, 5, new NodeResources(2, 4, 14, 16))))); Application original = new Application(ApplicationId.from("myTenant", "myApplication", "myInstance"), clusters); @@ -49,6 +51,7 @@ public class ApplicationSerializerTest { assertEquals(originalCluster.id(), serializedCluster.id()); assertEquals(originalCluster.minResources(), serializedCluster.minResources()); assertEquals(originalCluster.maxResources(), serializedCluster.maxResources()); + assertEquals(originalCluster.suggestedResources(), serializedCluster.suggestedResources()); assertEquals(originalCluster.targetResources(), serializedCluster.targetResources()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java index 7e9c3ef09dc..3a9d87b9db6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.node.Agent; import org.junit.Test; import java.util.Collection; @@ -23,6 +24,7 @@ import java.util.stream.Collectors; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; import static com.yahoo.config.provision.NodeResources.StorageType.local; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -159,7 +161,7 @@ public class InPlaceResizeProvisionTest { /** In this scenario there should be no resizing */ @Test public void increase_size_decrease_resources() { - addParentHosts(12, largeResources.with(fast)); + addParentHosts(14, largeResources.with(fast)); NodeResources resources = new NodeResources(4, 8, 16, 1); NodeResources halvedResources = new NodeResources(2, 4, 8, 1); @@ -176,6 +178,25 @@ public class InPlaceResizeProvisionTest { new PrepareHelper(tester, app).prepare(container1, 8, 1, halvedResources).activate(); assertSizeAndResources(listCluster(container1).retired(), 4, resources); assertSizeAndResources(listCluster(container1).not().retired(), 8, halvedResources); + + // Failing one of the new nodes should cause another new node to be allocated rather than + // unretiring one of the existing nodes, to avoid resizing during unretiring + Node nodeToFail = listCluster(container1).not().retired().asList().get(0); + tester.nodeRepository().fail(nodeToFail.hostname(), Agent.system, "testing"); + new PrepareHelper(tester, app).prepare(container1, 8, 1, halvedResources).activate(); + assertFalse(listCluster(container1).stream().anyMatch(n -> n.equals(nodeToFail))); + assertSizeAndResources(listCluster(container1).retired(), 4, resources); + assertSizeAndResources(listCluster(container1).not().retired(), 8, halvedResources); + + // ... same with setting a node to want to retire + System.out.println("marking wantToRetire"); + Node nodeToWantoToRetire = listCluster(container1).not().retired().asList().get(0); + tester.nodeRepository().write(nodeToWantoToRetire.with(nodeToWantoToRetire.status().withWantToRetire(true)), + tester.nodeRepository().lock(nodeToWantoToRetire)); + new PrepareHelper(tester, app).prepare(container1, 8, 1, halvedResources).activate(); + assertTrue(listCluster(container1).retired().stream().anyMatch(n -> n.equals(nodeToWantoToRetire))); + assertEquals(5, listCluster(container1).retired().size()); + assertSizeAndResources(listCluster(container1).not().retired(), 8, halvedResources); } @Test(expected = OutOfCapacityException.class) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java index 1265961e351..050f3b7e865 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/MultigroupProvisioningTest.java @@ -109,7 +109,7 @@ public class MultigroupProvisioningTest { } @Test - public void test_one_node_and_group_to_two_with_flavor_migration() { + public void test_one_node_and_group_to_two_with_resource_change() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.perf, RegionName.from("us-east"))).build(); ApplicationId application1 = tester.makeApplicationId(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 8fc73d68785..463a3ef3fb9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -59,7 +59,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); ApplicationId application2 = tester.makeApplicationId(); - tester.makeReadyNodes(21, defaultResources); + tester.makeReadyHosts(21, defaultResources).deployZoneApp(); // deploy SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); @@ -137,7 +137,7 @@ public class ProvisioningTest { public void nodeVersionIsReturnedIfSet() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); // deploy @@ -162,7 +162,7 @@ public class ProvisioningTest { public void dockerImageRepoIsReturnedIfSet() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); // deploy @@ -192,7 +192,8 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(24, defaultResources); + tester.makeReadyHosts(24, defaultResources); + tester.deployZoneApp(); // deploy SystemState state1 = prepare(application1, 2, 2, 3, 3, defaultResources, tester); @@ -262,18 +263,18 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(12, small); + tester.makeReadyHosts(12, small); + tester.makeReadyHosts(16, large); + tester.deployZoneApp(); // deploy SystemState state1 = prepare(application1, 2, 2, 4, 4, small, tester); tester.activate(application1, state1.allHosts); - // redeploy with reduced size (to cause us to have retired nodes before switching flavor) + // redeploy with reduced size (to cause us to have retired nodes before switching resources) SystemState state2 = prepare(application1, 2, 2, 3, 3, small, tester); tester.activate(application1, state2.allHosts); - tester.makeReadyNodes(16, large); - // redeploy with increased sizes and new flavor SystemState state3 = prepare(application1, 3, 4, 4, 5, large, tester); assertEquals("New nodes are reserved", 16, tester.nodeRepository().getNodes(application1, Node.State.reserved).size()); @@ -292,7 +293,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(5, defaultResources); + tester.makeReadyHosts(5, defaultResources).deployZoneApp(); // deploy SystemState state1 = prepare(application1, 2, 0, 3, 0, defaultResources, tester); @@ -315,7 +316,7 @@ public class ProvisioningTest { public void dev_deployment_node_size() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -327,8 +328,8 @@ public class ProvisioningTest { @Test public void requested_resources_info_is_retained() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + tester.makeReadyHosts(13, defaultResources).deployZoneApp(); - tester.makeReadyNodes(13, defaultResources, NodeType.host, 1); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -371,7 +372,7 @@ public class ProvisioningTest { public void deploy_specific_vespa_version() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -384,7 +385,7 @@ public class ProvisioningTest { public void deploy_specific_vespa_version_and_docker_image() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources, NodeType.host, 1); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -399,18 +400,69 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(4, defaultResources); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); } + @Test + public void test_changing_limits() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + tester.makeReadyHosts(30, new NodeResources(20, 40, 100, 4)).deployZoneApp(); + + ApplicationId app1 = tester.makeApplicationId("app1"); + ClusterSpec cluster1 = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("7").build(); + + // Initial deployment + tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 20), + resources(8, 4, 4, 20, 40))); + tester.assertNodes("Initial allocation at min", + 4, 2, 2, 10, 20, + app1, cluster1); + + // Move window above current allocation + tester.activate(app1, cluster1, Capacity.from(resources(8, 4, 4, 20, 40), + resources(10, 5, 5, 25, 50))); + tester.assertNodes("New allocation at new min", + 8, 4, 4, 20, 40, + app1, cluster1); + + // Move window below current allocation + tester.activate(app1, cluster1, Capacity.from(resources(4, 2, 2, 10, 20), + resources(6, 3, 3, 15, 25))); + tester.assertNodes("New allocation at new max", + 6, 3, 3, 15, 25, + app1, cluster1); + + // Widening window does not change allocation + tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 1, 5, 15), + resources(8, 4, 4, 20, 30))); + tester.assertNodes("Same allocation", + 6, 3, 3, 15, 25, + app1, cluster1); + + // Changing limits in opposite directions cause a mixture of min and max + tester.activate(app1, cluster1, Capacity.from(resources(2, 1, 10, 30, 5), + resources(4, 2, 14, 40, 10))); + tester.assertNodes("A mix of min and max", + 4, 2, 10, 30, 10, + app1, cluster1); + + // Changing group size + tester.activate(app1, cluster1, Capacity.from(resources(6, 3, 8, 25, 5), + resources(9, 3, 12, 35, 15))); + tester.assertNodes("Groups changed", + 6, 3, 10, 30, 10, + app1, cluster1); + } + @Test(expected = IllegalArgumentException.class) public void prod_deployment_requires_redundancy() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); prepare(application, 1, 2, 3, 3, defaultResources, tester); } @@ -419,7 +471,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); try { prepare(application, 2, 2, 3, 3, new NodeResources(2, 2, 10, 2), tester); @@ -434,7 +486,7 @@ public class ProvisioningTest { public void dev_deployment_flavor() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, new NodeResources(2, 4, 10, 2), NodeType.host, 1); + tester.makeReadyHosts(4, new NodeResources(2, 4, 10, 2)); tester.prepareAndActivateInfraApplication(tester.makeApplicationId(), NodeType.host); ApplicationId application = tester.makeApplicationId(); @@ -452,7 +504,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.test, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(4, large); + tester.makeReadyHosts(4, large).deployZoneApp(); SystemState state = prepare(application, 2, 2, 3, 3, large, tester); assertEquals(4, state.allHosts.size()); tester.activate(application, state.allHosts); @@ -463,7 +515,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.staging, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(14, defaultResources); + tester.makeReadyHosts(14, defaultResources).deployZoneApp(); SystemState state = prepare(application, 1, 1, 1, 64, defaultResources, tester); // becomes 1, 1, 1, 1, 6 assertEquals(9, state.allHosts.size()); tester.activate(application, state.allHosts); @@ -473,7 +525,7 @@ public class ProvisioningTest { public void activate_after_reservation_timeout() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); SystemState state = prepare(application, 2, 2, 3, 3, defaultResources, tester); @@ -495,7 +547,7 @@ public class ProvisioningTest { public void out_of_capacity() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(9, defaultResources); // need 2+2+3+3=10 + tester.makeReadyHosts(9, defaultResources).deployZoneApp(); // need 2+2+3+3=10 ApplicationId application = tester.makeApplicationId(); try { prepare(application, 2, 2, 3, 3, defaultResources, tester); @@ -512,7 +564,7 @@ public class ProvisioningTest { Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(13, defaultResources); + tester.makeReadyHosts(13, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); try { prepare(application, 2, 2, 6, 3, defaultResources, tester); @@ -530,7 +582,7 @@ public class ProvisioningTest { Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(13, defaultResources); + tester.makeReadyHosts(13, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); prepare(application, 2, 2, 6, 3, defaultResources, tester); } @@ -538,7 +590,7 @@ public class ProvisioningTest { @Test public void out_of_capacity_but_cannot_fail() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); - tester.makeReadyNodes(4, defaultResources); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); ApplicationId application = tester.makeApplicationId(); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); tester.prepare(application, cluster, Capacity.from(new ClusterResources(5, 1, NodeResources.unspecified), false, false)); @@ -572,7 +624,7 @@ public class ProvisioningTest { ApplicationId application = tester.makeApplicationId(); // Create 10 nodes - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); // Allocate 5 nodes ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); tester.activate(application, tester.prepare(application, cluster, capacity)); @@ -603,7 +655,7 @@ public class ProvisioningTest { ApplicationId application1 = tester.makeApplicationId(); - tester.makeReadyNodes(14, defaultResources); + tester.makeReadyHosts(14, defaultResources).deployZoneApp(); // deploy SystemState state1 = prepare(application1, 3, 3, 4, 4, defaultResources, tester); @@ -631,7 +683,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(10, defaultResources); + tester.makeReadyHosts(10, defaultResources).deployZoneApp(); // Deploy application { @@ -659,7 +711,7 @@ public class ProvisioningTest { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); ApplicationId application = tester.makeApplicationId(); - tester.makeReadyNodes(2, defaultResources); + tester.makeReadyHosts(2, defaultResources).deployZoneApp(); // Deploy fails with out of capacity try { @@ -670,7 +722,7 @@ public class ProvisioningTest { tester.getNodes(application, Node.State.reserved).size()); // Enough nodes become available - tester.makeReadyNodes(2, defaultResources); + tester.makeReadyHosts(2, defaultResources).deployZoneApp(); // Deploy is retried after a few minutes tester.clock().advance(Duration.ofMinutes(2)); @@ -740,7 +792,7 @@ public class ProvisioningTest { var tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); var application = tester.makeApplicationId(); - tester.makeReadyNodes(4, defaultResources); + tester.makeReadyHosts(4, defaultResources).deployZoneApp(); // Application allocates two content nodes initially, with cluster type content ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("1.2.3").build(); @@ -920,4 +972,8 @@ public class ProvisioningTest { .isPresent(); } + private ClusterResources resources(int nodes, int groups, double vcpu, double memory, double disk) { + return new ClusterResources(nodes, groups, new NodeResources(vcpu, memory, disk, 0.1)); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index a8df47aab1a..cbb15311867 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -149,6 +149,10 @@ public class ProvisioningTester { return hosts1; } + public Collection<HostSpec> activate(ApplicationId application, ClusterSpec cluster, Capacity capacity) { + return activate(application, prepare(application, cluster, capacity, true)); + } + public Collection<HostSpec> activate(ApplicationId application, Collection<HostSpec> hosts) { NestedTransaction transaction = new NestedTransaction(); transaction.add(new CuratorTransaction(curator)); @@ -197,6 +201,22 @@ public class ProvisioningTester { } } + /** Assert on the current *non retired* nodes */ + public void assertNodes(String explanation, int nodes, int groups, double vcpu, double memory, double disk, + ApplicationId app, ClusterSpec cluster) { + List<Node> nodeList = nodeRepository.list().owner(app).cluster(cluster.id()).not().retired().asList(); + assertEquals(explanation + ": Node count", + nodes, + nodeList.size()); + assertEquals(explanation + ": Group count", + groups, + nodeList.stream().map(n -> n.allocation().get().membership().cluster().group().get()).distinct().count()); + for (Node node : nodeList) + assertEquals(explanation + ": Resources", + new NodeResources(vcpu, memory, disk, 0.1), + node.flavor().resources()); + } + public void fail(HostSpec host) { int beforeFailCount = nodeRepository.getNode(host.hostname(), Node.State.active).get().status().failCount(); Node failedNode = nodeRepository.fail(host.hostname(), Agent.system, "Failing to unit test"); @@ -241,6 +261,12 @@ public class ProvisioningTester { return makeReadyNodes(n, flavor, NodeType.tenant); } + /** Call deployZoneApp() after this before deploying applications */ + public ProvisioningTester makeReadyHosts(int n, NodeResources resources) { + makeReadyNodes(n, resources, NodeType.host, 5); + return this; + } + public List<Node> makeReadyNodes(int n, NodeResources resources) { return makeReadyNodes(n, resources, NodeType.tenant, 0); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java new file mode 100644 index 00000000000..e9811985b7d --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java @@ -0,0 +1,30 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.application.container.handler.Request; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class LoadBalancersV1ApiTest { + + private RestApiTester tester; + + @Before + public void createTester() { + tester = new RestApiTester(); + } + + @After + public void closeTester() { + tester.close(); + } + + @Test + public void test_load_balancers() throws Exception { + tester.assertFile(new Request("http://localhost:8080/loadbalancers/v1/"), "load-balancers.json"); + tester.assertFile(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant4.application4.instance4"), "load-balancers-single.json"); + tester.assertResponse(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant.nonexistent.default"), "{\"loadBalancers\":[]}"); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializerTest.java index 05b23addddf..9bc27c0cc04 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodeSerializerTest.java @@ -1,9 +1,10 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.restapi.NodeSerializer; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -13,19 +14,17 @@ import static org.junit.Assert.assertEquals; */ public class NodeSerializerTest { - private final NodeSerializer serializer = new NodeSerializer(); - @Test public void serialize_node_types() { for (NodeType t : NodeType.values()) { - assertEquals(t, serializer.typeFrom(serializer.toString(t))); + assertEquals(t, NodeSerializer.typeFrom(NodeSerializer.toString(t))); } } @Test public void serialize_node_states() { for (Node.State s : Node.State.values()) { - assertEquals(s, serializer.stateFrom(serializer.toString(s))); + assertEquals(s, NodeSerializer.stateFrom(NodeSerializer.toString(s))); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 0001c344dd3..c16a3f3b3e0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/RestApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -1,5 +1,5 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.restapi.v2; +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.application.Networking; import com.yahoo.application.container.JDisc; @@ -40,23 +40,21 @@ import static org.junit.Assert.assertFalse; * * @author bratseth */ -public class RestApiTest { +public class NodesV2ApiTest { - private final static String responsesPath = "src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/"; - - private JDisc container; + private RestApiTester tester; @Before - public void startContainer() { - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); + public void createTester() { + tester = new RestApiTester(); } @After - public void stopContainer() { - if (container != null) container.close(); + public void closeTester() { + tester.close(); } - /** This test gives examples of all the requests that can be made to nodes/v2 */ + /** This test gives examples of the node requests that can be made to nodes/v2 */ @Test public void test_requests() throws Exception { // GET @@ -82,8 +80,8 @@ public class RestApiTest { new byte[0], Request.Method.POST)); assertRestart(11, new Request("http://localhost:8080/nodes/v2/command/restart", new byte[0], Request.Method.POST)); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), - "\"restartGeneration\":3"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), + "\"restartGeneration\":3"); // POST reboot command assertReboot(12, new Request("http://localhost:8080/nodes/v2/command/reboot?state=failed%20active", @@ -92,28 +90,28 @@ public class RestApiTest { new byte[0], Request.Method.POST)); assertReboot(19, new Request("http://localhost:8080/nodes/v2/command/reboot", new byte[0], Request.Method.POST)); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), - "\"rebootGeneration\":4"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), + "\"rebootGeneration\":4"); // POST new nodes assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.8.1") + "," + // test with only 1 ip address - asNodeJson("host9.yahoo.com", "large-variant", "127.0.9.1", "::9:1") + "," + - asHostJson("parent2.yahoo.com", "large-variant", Optional.of(TenantName.from("myTenant")), "127.0.127.1", "::127:1") + "," + - asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com", "::11") + "]"). + asNodeJson("host9.yahoo.com", "large-variant", "127.0.9.1", "::9:1") + "," + + asHostJson("parent2.yahoo.com", "large-variant", Optional.of(TenantName.from("myTenant")), "127.0.127.1", "::127:1") + "," + + asDockerNodeJson("host11.yahoo.com", "parent.host.yahoo.com", "::11") + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST), - "{\"message\":\"Added 4 nodes to the provisioned state\"}"); + "{\"message\":\"Added 4 nodes to the provisioned state\"}"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), "node8.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com"), "node9.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host11.yahoo.com"), "node11.json"); assertFile(new Request("http://localhost:8080/nodes/v2/node/parent2.yahoo.com"), "parent2.json"); // POST duplicate node - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.8") + "]").getBytes(StandardCharsets.UTF_8), - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add provisioned host host8.yahoo.com: A node with this name already exists\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.8") + "]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add provisioned host host8.yahoo.com: A node with this name already exists\"}"); // DELETE a provisioned node assertResponse(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com", @@ -127,8 +125,8 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved host8.yahoo.com to ready\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), - "\"state\":\"ready\""); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com"), + "\"state\":\"ready\""); // calling ready again is a noop: assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host8.yahoo.com", new byte[0], Request.Method.PUT), @@ -138,8 +136,8 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host2.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved host2.yahoo.com to failed\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), - "\"state\":\"failed\""); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com"), + "\"state\":\"failed\""); // ... and put it back in active (after fixing). This is useful to restore data when multiple nodes fail. assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host2.yahoo.com", new byte[0], Request.Method.PUT), @@ -149,8 +147,8 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/host8.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved host8.yahoo.com to parked\"}"); - assertResponseContains(new Request("http://localhost:8080()/nodes/v2/node/host8.yahoo.com"), - "\"state\":\"parked\""); + tester.assertResponseContains(new Request("http://localhost:8080()/nodes/v2/node/host8.yahoo.com"), + "\"state\":\"parked\""); // ... and delete it assertResponse(new Request("http://localhost:8080/nodes/v2/node/host8.yahoo.com", new byte[0], Request.Method.DELETE), @@ -160,8 +158,8 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/test-node-pool-102-2", new byte[0], Request.Method.PUT), "{\"message\":\"Moved test-node-pool-102-2 to failed\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), - "\"state\":\"failed\""); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2"), + "\"state\":\"failed\""); // ... and deallocate it such that it moves to dirty and is recycled assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/test-node-pool-102-2", new byte[0], Request.Method.PUT), @@ -172,10 +170,10 @@ public class RestApiTest { new byte[0], Request.Method.PUT), "{\"message\":\"Moved test-node-pool-102-2 to ready\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", new byte[0], Request.Method.GET), - 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'test-node-pool-102-2'\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", new byte[0], Request.Method.GET), + 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'test-node-pool-102-2'\"}"); - // Put a host in failed and make sure it's children are also failed + // Put a host in failed and make sure its children are also failed assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/dockerhost1.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved dockerhost1.yahoo.com, host4.yahoo.com to failed\"}"); @@ -218,17 +216,37 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"wantToDeprovision\": true}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\""); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "\"modelName\":\"foo\""); assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", Utf8.toBytes("{\"modelName\": null}"), Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); - assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "modelName", false); - container.handleRequest((new Request("http://localhost:8080/nodes/v2/upgrade/tenant", Utf8.toBytes("{\"dockerImage\": \"docker.domain.tld/my/image\"}"), Request.Method.PATCH))); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "modelName", false); + tester.container().handleRequest((new Request("http://localhost:8080/nodes/v2/upgrade/tenant", Utf8.toBytes("{\"dockerImage\": \"docker.domain.tld/my/image\"}"), Request.Method.PATCH))); - ((OrchestratorMock) container.components().getComponent(OrchestratorMock.class.getName())) + ((OrchestratorMock) tester.container().components().getComponent(OrchestratorMock.class.getName())) .suspend(new HostName("host4.yahoo.com")); assertFile(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "node4-after-changes.json"); + + // move the docker host to deprovisioned + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + new byte[0], Request.Method.DELETE), + "{\"message\":\"Removed dockerhost1.yahoo.com\"}"); + // ... and then forget it completely + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + new byte[0], Request.Method.DELETE), + "{\"message\":\"Permanently removed dockerhost1.yahoo.com\"}"); + + + } + + @Test + public void test_application_requests() throws Exception { + assertFile(new Request("http://localhost:8080/nodes/v2/application/"), "applications.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/application/tenant1.application1.instance1"), + "application1.json"); + assertFile(new Request("http://localhost:8080/nodes/v2/application/tenant2.application2.instance2"), + "application2.json"); } @Test @@ -251,10 +269,10 @@ public class RestApiTest { "{\"message\":\"Executed job 'PeriodicApplicationMaintainer'\"}"); // POST run of unknown maintenance job - assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/run/foo", - new byte[0], Request.Method.POST), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No such job 'foo'\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/maintenance/run/foo", + new byte[0], Request.Method.POST), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No such job 'foo'\"}"); } @Test @@ -270,7 +288,8 @@ public class RestApiTest { Request req = new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", Utf8.toBytes("{\"currentRestartGeneration\": 1}"), Request.Method.POST); req.getHeaders().add("X-HTTP-Method-Override", "GET"); - assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Illegal X-HTTP-Method-Override header for POST request. Accepts 'PATCH' but got 'GET'\"}"); + tester.assertResponse(req, 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Illegal X-HTTP-Method-Override header for POST request. Accepts 'PATCH' but got 'GET'\"}"); } @Test @@ -298,56 +317,56 @@ public class RestApiTest { ("[" + asNodeJson("host-with-ip.yahoo.com", "default", "foo") + "]"). getBytes(StandardCharsets.UTF_8), Request.Method.POST); - assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Found one or more invalid addresses in [foo]: 'foo' is not an IP string literal.\"}"); + tester.assertResponse(req, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Found one or more invalid addresses in [foo]: 'foo' is not an IP string literal.\"}"); // Attempt to POST tenant node with already assigned IP - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("tenant-node-foo.yahoo.com", "default", "127.0.1.1") + "]", - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.1.1] to tenant-node-foo.yahoo.com: [127.0.1.1] already assigned to host1.yahoo.com\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asNodeJson("tenant-node-foo.yahoo.com", "default", "127.0.1.1") + "]", + Request.Method.POST), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.1.1] to tenant-node-foo.yahoo.com: [127.0.1.1] already assigned to host1.yahoo.com\"}"); // Attempt to PATCH existing tenant node with already assigned IP - assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", - "{\"ipAddresses\": [\"127.0.2.1\"]}", - Request.Method.PATCH), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [127.0.2.1] to test-node-pool-102-2: [127.0.2.1] already assigned to host2.yahoo.com\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", + "{\"ipAddresses\": [\"127.0.2.1\"]}", + Request.Method.PATCH), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [127.0.2.1] to test-node-pool-102-2: [127.0.2.1] already assigned to host2.yahoo.com\"}"); // Attempt to POST host node with already assigned IP - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asHostJson("host200.yahoo.com", "default", Optional.empty(), "127.0.2.1") + "]", - Request.Method.POST), 400, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asHostJson("host200.yahoo.com", "default", Optional.empty(), "127.0.2.1") + "]", + Request.Method.POST), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.2.1] to host200.yahoo.com: [127.0.2.1] already assigned to host2.yahoo.com\"}"); // Attempt to PATCH host node with IP in the pool of another node - assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", - "{\"ipAddresses\": [\"::104:3\"]}", - Request.Method.PATCH), 400, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + "{\"ipAddresses\": [\"::104:3\"]}", + Request.Method.PATCH), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [::100:4, ::100:3, ::100:2, ::104:3] to dockerhost1.yahoo.com: [::104:3] already assigned to dockerhost5.yahoo.com\"}"); // Node types running a single container can share their IP address with child node - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.42.1") + "]", - Request.Method.POST), 200, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asNodeJson("cfghost42.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.42.1") + "]", + Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asDockerNodeJson("cfg42.yahoo.com", NodeType.config, "cfghost42.yahoo.com", "127.0.42.1") + "]", - Request.Method.POST), 200, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asDockerNodeJson("cfg42.yahoo.com", NodeType.config, "cfghost42.yahoo.com", "127.0.42.1") + "]", + Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); // ... but cannot share with child node of wrong type - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asDockerNodeJson("proxy42.yahoo.com", NodeType.proxy, "cfghost42.yahoo.com", "127.0.42.1") + "]", - Request.Method.POST), 400, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asDockerNodeJson("proxy42.yahoo.com", NodeType.proxy, "cfghost42.yahoo.com", "127.0.42.1") + "]", + Request.Method.POST), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot assign [127.0.42.1] to proxy42.yahoo.com: [127.0.42.1] already assigned to cfg42.yahoo.com\"}"); // ... nor with child node on different host - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.43.1") + "]", - Request.Method.POST), 200, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + "[" + asNodeJson("cfghost43.yahoo.com", NodeType.confighost, "default", Optional.empty(), "127.0.43.1") + "]", + Request.Method.POST), 200, "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/cfg42.yahoo.com", - "{\"ipAddresses\": [\"127.0.43.1\"]}", - Request.Method.PATCH), 400, + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/cfg42.yahoo.com", + "{\"ipAddresses\": [\"127.0.43.1\"]}", + Request.Method.PATCH), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'ipAddresses': Cannot assign [127.0.43.1] to cfg42.yahoo.com: [127.0.43.1] already assigned to cfghost43.yahoo.com\"}"); } @@ -376,10 +395,10 @@ public class RestApiTest { Utf8.toBytes("{\"reports\":{\"diskSpace\":{\"createdMillis\":2,\"description\":\"" + msg + "\",\"type\": \"HARD_FAIL\"}}}"), Request.Method.PATCH), "{\"message\":\"Updated host12.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host12.yahoo.com", new byte[0], Request.Method.PUT), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"provisioned host host12.yahoo.com cannot be readied because it has " + - "hard failures: [diskSpace reported 1970-01-01T00:00:00.002Z: " + msg + "]\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host12.yahoo.com", new byte[0], Request.Method.PUT), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"provisioned host host12.yahoo.com cannot be readied because it has " + + "hard failures: [diskSpace reported 1970-01-01T00:00:00.002Z: " + msg + "]\"}"); } @Test @@ -395,13 +414,13 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/foo.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved foo.yahoo.com to dirty\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com"), - "\"rebootGeneration\":1"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com"), + "\"rebootGeneration\":1"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com", Utf8.toBytes("{\"currentRebootGeneration\": 42}"), Request.Method.PATCH), "{\"message\":\"Updated foo.yahoo.com\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com"), - "\"rebootGeneration\":1"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/foo.yahoo.com"), + "\"rebootGeneration\":1"); } @Test @@ -427,21 +446,21 @@ public class RestApiTest { @Test public void test_invalid_requests() throws Exception { - assertResponse(new Request("http://localhost:8080/nodes/v2/node/node-does-not-exist", - new byte[0], Request.Method.GET), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/node-does-not-exist", + new byte[0], Request.Method.GET), 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'node-does-not-exist'\"}"); // Attempt to fail and ready an allocated node without going through dirty - assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/node-does-not-exist", - new byte[0], Request.Method.PUT), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/node-does-not-exist", + new byte[0], Request.Method.PUT), 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not move node-does-not-exist to failed: Node not found\"}"); // Attempt to fail and ready an allocated node without going through dirty assertResponse(new Request("http://localhost:8080/nodes/v2/state/failed/host1.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved host1.yahoo.com to failed\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host1.yahoo.com", - new byte[0], Request.Method.PUT), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host1.yahoo.com", + new byte[0], Request.Method.PUT), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot make failed host host1.yahoo.com allocated to tenant1.application1.instance1 as 'container/id1/0/0' available for new allocation as it is not in state [dirty]\"}"); @@ -457,8 +476,8 @@ public class RestApiTest { assertResponse(new Request("http://localhost:8080/nodes/v2/state/parked/host2.yahoo.com", new byte[0], Request.Method.PUT), "{\"message\":\"Moved host2.yahoo.com to parked\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host2.yahoo.com", - new byte[0], Request.Method.PUT), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/ready/host2.yahoo.com", + new byte[0], Request.Method.PUT), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot make parked host host2.yahoo.com allocated to tenant2.application2.instance2 as 'content/id2/0/0' available for new allocation as it is not in state [dirty]\"}"); // (... while dirty then ready works (the ready move will be initiated by node maintenance)) assertResponse(new Request("http://localhost:8080/nodes/v2/state/dirty/host2.yahoo.com", @@ -469,42 +488,42 @@ public class RestApiTest { "{\"message\":\"Moved host2.yahoo.com to ready\"}"); // Attempt to DELETE a node which has been removed - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com", - new byte[0], Request.Method.DELETE), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host2.yahoo.com", + new byte[0], Request.Method.DELETE), 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node with hostname 'host2.yahoo.com'\"}"); // Attempt to DELETE allocated node - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - new byte[0], Request.Method.DELETE), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + new byte[0], Request.Method.DELETE), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"active child node host4.yahoo.com allocated to tenant3.application3.instance3 as 'content/id3/0/0' is currently allocated and cannot be removed\"}"); // PUT current restart generation with string instead of long - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"currentRestartGeneration\": \"1\"}"), Request.Method.PATCH), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": \"1\"}"), Request.Method.PATCH), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Expected a LONG value, got a STRING\"}"); // PUT flavor with long instead of string - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", - Utf8.toBytes("{\"flavor\": 1}"), Request.Method.PATCH), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com", + Utf8.toBytes("{\"flavor\": 1}"), Request.Method.PATCH), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'flavor': Expected a STRING value, got a LONG\"}"); // Attempt to set nonexisting node to active - assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host2.yahoo.com", - new byte[0], Request.Method.PUT), 404, - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not move host2.yahoo.com to active: Node not found\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/state/active/host2.yahoo.com", + new byte[0], Request.Method.PUT), 404, + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not move host2.yahoo.com to active: Node not found\"}"); // Attempt to POST duplicate nodes - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.1", "::254:1") + "," + - asNodeJson("host8.yahoo.com", "large-variant", "127.0.253.1", "::253:1") + "]").getBytes(StandardCharsets.UTF_8), - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add nodes: provisioned host host8.yahoo.com is duplicated in the argument list\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.1", "::254:1") + "," + + asNodeJson("host8.yahoo.com", "large-variant", "127.0.253.1", "::253:1") + "]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add nodes: provisioned host host8.yahoo.com is duplicated in the argument list\"}"); // Attempt to PATCH field not relevant for child node - assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", - Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'modelName': A child node cannot have model name set\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/test-node-pool-102-2", + Utf8.toBytes("{\"modelName\": \"foo\"}"), Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'modelName': A child node cannot have model name set\"}"); } @Test @@ -522,15 +541,16 @@ public class RestApiTest { Request.Method.PATCH), "{\"message\":\"Updated host4.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/doesnotexist.yahoo.com", - Utf8.toBytes("{\"currentRestartGeneration\": 1}"), - Request.Method.PATCH), - 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node found with hostname doesnotexist.yahoo.com\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/doesnotexist.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), + Request.Method.PATCH), + 404, "{\"error-code\":\"NOT_FOUND\",\"message\":\"No node found with hostname doesnotexist.yahoo.com\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com", - Utf8.toBytes("{\"currentRestartGeneration\": 1}"), - Request.Method.PATCH), - 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Node is not allocated\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host5.yahoo.com", + Utf8.toBytes("{\"currentRestartGeneration\": 1}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'currentRestartGeneration': Node is not allocated\"}"); } @Test @@ -575,25 +595,25 @@ public class RestApiTest { assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports.json"); // Patching with an empty reports is no-op - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", - Utf8.toBytes("{\"reports\": {}}"), - Request.Method.PATCH), - 200, - "{\"message\":\"Updated host6.yahoo.com\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + Utf8.toBytes("{\"reports\": {}}"), + Request.Method.PATCH), + 200, + "{\"message\":\"Updated host6.yahoo.com\"}"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports.json"); // Patching existing report overwrites - assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", - Utf8.toBytes("{" + - " \"reports\": {" + - " \"actualCpuCores\": {" + - " \"createdMillis\": 3 " + - " }" + - " }" + - "}"), - Request.Method.PATCH), - 200, - "{\"message\":\"Updated host6.yahoo.com\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com", + Utf8.toBytes("{" + + " \"reports\": {" + + " \"actualCpuCores\": {" + + " \"createdMillis\": 3 " + + " }" + + " }" + + "}"), + Request.Method.PATCH), + 200, + "{\"message\":\"Updated host6.yahoo.com\"}"); assertFile(new Request("http://localhost:8080/nodes/v2/node/host6.yahoo.com"), "node6-reports-2.json"); // Clearing one report @@ -636,26 +656,26 @@ public class RestApiTest { "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.456\",\"controller\":\"6.123.456\"},\"osVersions\":{},\"dockerImages\":{}}"); // Setting version for unsupported node type fails - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/tenant", - Utf8.toBytes("{\"version\": \"6.123.456\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Target version for type tenant is not allowed\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/tenant", + Utf8.toBytes("{\"version\": \"6.123.456\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Target version for type tenant is not allowed\"}"); // Omitting version field fails - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"At least one of 'version', 'osVersion' or 'dockerImage' must be set\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"At least one of 'version', 'osVersion' or 'dockerImage' must be set\"}"); // Downgrade without force fails - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"version\": \"6.123.1\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. " + - "Current target version: 6.123.456, attempted to set target version: 6.123.1\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": \"6.123.1\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. " + + "Current target version: 6.123.456, attempted to set target version: 6.123.1\"}"); // Downgrade with force is OK assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", @@ -668,11 +688,11 @@ public class RestApiTest { "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.123.1\",\"controller\":\"6.123.456\"},\"osVersions\":{},\"dockerImages\":{}}"); // Setting empty version without force fails - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"version\": null}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. Current target version: 6.123.1, attempted to set target version: 0.0.0\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"version\": null}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade version without setting 'force'. Current target version: 6.123.1, attempted to set target version: 0.0.0\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", Utf8.toBytes("{\"version\": null, \"force\": true}"), @@ -704,18 +724,18 @@ public class RestApiTest { "{\"message\":\"Set version to 6.124.42, osVersion to 7.5.2 for nodes of type confighost\"}"); // Attempt to upgrade unsupported node type - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", - Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Node type 'config' does not support OS upgrades\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/config", + Utf8.toBytes("{\"osVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Node type 'config' does not support OS upgrades\"}"); // Attempt to downgrade OS - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"osVersion\": \"7.4.2\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot set target OS version to 7.4.2 without setting 'force', as it's lower than the current version: 7.5.2\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": \"7.4.2\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot set target OS version to 7.4.2 without setting 'force', as it's lower than the current version: 7.5.2\"}"); // Downgrading OS with force succeeds assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", @@ -724,11 +744,11 @@ public class RestApiTest { "{\"message\":\"Set osVersion to 7.4.2 for nodes of type confighost\"}"); // Current target is considered bad, remove it - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"osVersion\": null}"), - Request.Method.PATCH), - 200, - "{\"message\":\"Set osVersion to null for nodes of type confighost\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"osVersion\": null}"), + Request.Method.PATCH), + 200, + "{\"message\":\"Set osVersion to null for nodes of type confighost\"}"); // Set docker image for config and tenant assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/tenant", @@ -744,11 +764,11 @@ public class RestApiTest { "{\"versions\":{\"config\":\"6.123.456\",\"confighost\":\"6.124.42\",\"controller\":\"6.123.456\"},\"osVersions\":{\"host\":\"7.5.2\"},\"dockerImages\":{\"tenant\":\"my-repo.my-domain.example:1234/repo/tenant\",\"config\":\"my-repo.my-domain.example:1234/repo/image\"}}"); // Cannot set docker image for non docker node type - assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", - Utf8.toBytes("{\"dockerImage\": \"my-repo.my-domain.example:1234/repo/image\"}"), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Setting docker image for confighost nodes is unsupported\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/upgrade/confighost", + Utf8.toBytes("{\"dockerImage\": \"my-repo.my-domain.example:1234/repo/image\"}"), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Setting docker image for confighost nodes is unsupported\"}"); } @Test @@ -760,12 +780,12 @@ public class RestApiTest { "{\"message\":\"Set osVersion to 7.5.2 for nodes of type host\"}"); // Activate target - var nodeRepository = (NodeRepository) container.components().getComponent(MockNodeRepository.class.getName()); + var nodeRepository = (NodeRepository)tester.container().components().getComponent(MockNodeRepository.class.getName()); var osUpgradeActivator = new OsUpgradeActivator(nodeRepository, Duration.ofDays(1)); osUpgradeActivator.run(); // Other node type does not return wanted OS version - Response r = container.handleRequest(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com")); + Response r = tester.container().handleRequest(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com")); assertFalse("Response omits wantedOsVersions field", r.getBodyAsString().contains("wantedOsVersion")); // Node updates its node object after upgrading OS @@ -845,21 +865,13 @@ public class RestApiTest { } @Test - public void test_load_balancers() throws Exception { - assertFile(new Request("http://localhost:8080/loadbalancers/v1/"), "load-balancers.json"); - assertFile(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant4.application4.instance4"), "load-balancers-single.json"); - assertResponse(new Request("http://localhost:8080/loadbalancers/v1/?application=tenant.nonexistent.default"), "{\"loadBalancers\":[]}"); - } - - @Test public void test_flavor_overrides() throws Exception { String host = "parent2.yahoo.com"; // Test adding with overrides - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[{\"hostname\":\"" + host + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + - "\"flavor\":\"large-variant\",\"resources\":{\"diskGb\":1234,\"memoryGb\":4321}}]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[{\"hostname\":\"" + host + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + + "\"flavor\":\"large-variant\",\"resources\":{\"diskGb\":1234,\"memoryGb\":4321}}]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Can only override disk GB for configured flavor\"}"); @@ -869,8 +881,8 @@ public class RestApiTest { getBytes(StandardCharsets.UTF_8), Request.Method.POST), "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host), - "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":1234.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host), + "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":1234.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); // Test adding tenant node String tenant = "node-1-3.yahoo.com"; @@ -881,21 +893,21 @@ public class RestApiTest { getBytes(StandardCharsets.UTF_8), Request.Method.POST), "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + tenant), resources); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + tenant), resources); // Test patching with overrides - assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host, - "{\"minDiskAvailableGb\":5432,\"minMainMemoryAvailableGb\":2345}".getBytes(StandardCharsets.UTF_8), - Request.Method.PATCH), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'minMainMemoryAvailableGb': Can only override disk GB for configured flavor\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host, + "{\"minDiskAvailableGb\":5432,\"minMainMemoryAvailableGb\":2345}".getBytes(StandardCharsets.UTF_8), + Request.Method.PATCH), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'minMainMemoryAvailableGb': Can only override disk GB for configured flavor\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host, "{\"minDiskAvailableGb\":5432}".getBytes(StandardCharsets.UTF_8), Request.Method.PATCH), "{\"message\":\"Updated " + host + "\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host), - "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":5432.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host), + "\"resources\":{\"vcpu\":64.0,\"memoryGb\":128.0,\"diskGb\":5432.0,\"bandwidthGbps\":15.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); } @Test @@ -903,28 +915,27 @@ public class RestApiTest { String hostname = "node123.yahoo.com"; String resources = "\"resources\":{\"vcpu\":5.0,\"memoryGb\":4321.0,\"diskGb\":1234.0,\"bandwidthGbps\":0.3,\"diskSpeed\":\"slow\",\"storageType\":\"local\"}"; // Test adding new node with resources - assertResponse(new Request("http://localhost:8080/nodes/v2/node", - ("[{\"hostname\":\"" + hostname + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + - resources.replace("\"memoryGb\":4321.0,", "") + "}]"). - getBytes(StandardCharsets.UTF_8), - Request.Method.POST), - 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Required field 'memoryGb' is missing\"}"); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", + ("[{\"hostname\":\"" + hostname + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + + resources.replace("\"memoryGb\":4321.0,", "") + "}]").getBytes(StandardCharsets.UTF_8), + Request.Method.POST), + 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Required field 'memoryGb' is missing\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[{\"hostname\":\"" + hostname + "\"," + createIpAddresses("::1") + "\"openStackId\":\"osid-123\"," + resources + "}]") .getBytes(StandardCharsets.UTF_8), Request.Method.POST), "{\"message\":\"Added 1 nodes to the provisioned state\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), resources); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), resources); // Test patching with overrides assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + hostname, "{\"diskGb\":12,\"memoryGb\":34,\"vcpu\":56,\"fastDisk\":true,\"remoteStorage\":true,\"bandwidthGbps\":78.0}".getBytes(StandardCharsets.UTF_8), Request.Method.PATCH), "{\"message\":\"Updated " + hostname + "\"}"); - assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), - "\"resources\":{\"vcpu\":56.0,\"memoryGb\":34.0,\"diskGb\":12.0,\"bandwidthGbps\":78.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + hostname), + "\"resources\":{\"vcpu\":56.0,\"memoryGb\":34.0,\"diskGb\":12.0,\"bandwidthGbps\":78.0,\"diskSpeed\":\"fast\",\"storageType\":\"remote\"}"); } private static String asDockerNodeJson(String hostname, String parentHostname, String... ipAddress) { @@ -965,71 +976,20 @@ public class RestApiTest { "],"; } - /** Asserts a particular response and 200 as response status */ - private void assertResponse(Request request, String responseMessage) throws IOException { - assertResponse(request, 200, responseMessage); - } - - private void assertResponse(Request request, int responseStatus, String responseMessage) throws IOException { - Response response = container.handleRequest(request); - // Compare both status and message at once for easier diagnosis - assertEquals("status: " + responseStatus + "\nmessage: " + responseMessage, - "status: " + response.getStatus() + "\nmessage: " + response.getBodyAsString()); - } - - private void assertResponseContains(Request request, String responseSnippet) throws IOException { - assertPartialResponse(request, responseSnippet, true); - } - - private void assertPartialResponse(Request request, String responseSnippet, boolean match) throws IOException { - String response = container.handleRequest(request).getBodyAsString(); - assertEquals(String.format("Expected response to " + (match ? " " : "not ") + "contain: %s\nResponse: %s", - responseSnippet, response), match, response.contains(responseSnippet)); - } - - private void assertFile(Request request, String responseFile) throws IOException { - String expectedResponse = IOUtils.readFile(new File(responsesPath + responseFile)); - expectedResponse = include(expectedResponse); - expectedResponse = expectedResponse.replaceAll("(\"[^\"]*\")|\\s*", "$1"); // Remove whitespace - String responseString = container.handleRequest(request).getBodyAsString(); - if (expectedResponse.contains("(ignore)")) { - // Convert expected response to a literal pattern and replace any ignored field with a pattern that matches - // until the first stop character - String stopCharacters = "[^,:\\\\[\\\\]{}]"; - String expectedResponsePattern = Pattern.quote(expectedResponse) - .replaceAll("\"?\\(ignore\\)\"?", "\\\\E" + - stopCharacters + "*\\\\Q"); - if (!Pattern.matches(expectedResponsePattern, responseString)) { - throw new ComparisonFailure(responseFile + " (with ignored fields)", expectedResponsePattern, - responseString); - } - } else { - assertEquals(responseFile, expectedResponse, responseString); - } - } - private void assertRestart(int restartCount, Request request) throws IOException { - assertResponse(request, 200, "{\"message\":\"Scheduled restart of " + restartCount + " matching nodes\"}"); + tester.assertResponse(request, 200, "{\"message\":\"Scheduled restart of " + restartCount + " matching nodes\"}"); } private void assertReboot(int rebootCount, Request request) throws IOException { - assertResponse(request, 200, "{\"message\":\"Scheduled reboot of " + rebootCount + " matching nodes\"}"); + tester.assertResponse(request, 200, "{\"message\":\"Scheduled reboot of " + rebootCount + " matching nodes\"}"); + } + + private void assertFile(Request request, String file) throws IOException { + tester.assertFile(request, file); } - /** Replaces @include(localFile) with the content of the file */ - private String include(String response) throws IOException { - // Please don't look at this code - int includeIndex = response.indexOf("@include("); - if (includeIndex < 0) return response; - String prefix = response.substring(0, includeIndex); - String rest = response.substring(includeIndex + "@include(".length()); - int filenameEnd = rest.indexOf(")"); - String includeFileName = rest.substring(0, filenameEnd); - String includedContent = IOUtils.readFile(new File(responsesPath + includeFileName)); - includedContent = include(includedContent); - String postFix = rest.substring(filenameEnd + 1); - postFix = include(postFix); - return prefix + includedContent + postFix; + private void assertResponse(Request request, String file) throws IOException { + tester.assertResponse(request, file); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java new file mode 100644 index 00000000000..847a4e2500b --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java @@ -0,0 +1,96 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.application.Networking; +import com.yahoo.application.container.JDisc; +import com.yahoo.application.container.handler.Request; +import com.yahoo.application.container.handler.Response; +import com.yahoo.io.IOUtils; +import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; +import org.junit.ComparisonFailure; + +import java.io.File; +import java.io.IOException; +import java.util.regex.Pattern; + +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class RestApiTester { + + private final static String responsesPath = "src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/"; + + private final JDisc container; + + public RestApiTester() { + container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0), Networking.disable); + } + + public void close() { + if (container != null) container.close(); + } + + public JDisc container() { return container; } + + /** Asserts a particular response and 200 as response status */ + public void assertResponse(Request request, String responseMessage) throws IOException { + assertResponse(request, 200, responseMessage); + } + + public void assertResponse(Request request, int responseStatus, String responseMessage) throws IOException { + Response response = container.handleRequest(request); + // Compare both status and message at once for easier diagnosis + assertEquals("status: " + responseStatus + "\nmessage: " + responseMessage, + "status: " + response.getStatus() + "\nmessage: " + response.getBodyAsString()); + } + + public void assertResponseContains(Request request, String responseSnippet) throws IOException { + assertPartialResponse(request, responseSnippet, true); + } + + public void assertPartialResponse(Request request, String responseSnippet, boolean match) throws IOException { + String response = container.handleRequest(request).getBodyAsString(); + assertEquals(String.format("Expected response to " + (match ? " " : "not ") + "contain: %s\nResponse: %s", + responseSnippet, response), match, response.contains(responseSnippet)); + } + + public void assertFile(Request request, String responseFile) throws IOException { + String expectedResponse = IOUtils.readFile(new File(responsesPath + responseFile)); + expectedResponse = include(expectedResponse); + expectedResponse = expectedResponse.replaceAll("(\"[^\"]*\")|\\s*", "$1"); // Remove whitespace + String responseString = container.handleRequest(request).getBodyAsString(); + if (expectedResponse.contains("(ignore)")) { + // Convert expected response to a literal pattern and replace any ignored field with a pattern that matches + // until the first stop character + String stopCharacters = "[^,:\\\\[\\\\]{}]"; + String expectedResponsePattern = Pattern.quote(expectedResponse) + .replaceAll("\"?\\(ignore\\)\"?", "\\\\E" + + stopCharacters + "*\\\\Q"); + if (!Pattern.matches(expectedResponsePattern, responseString)) { + throw new ComparisonFailure(responseFile + " (with ignored fields)", expectedResponsePattern, + responseString); + } + } else { + assertEquals(responseFile, expectedResponse, responseString); + } + } + + /** Replaces @include(localFile) with the content of the file */ + public String include(String response) throws IOException { + // Please don't look at this code + int includeIndex = response.indexOf("@include("); + if (includeIndex < 0) return response; + String prefix = response.substring(0, includeIndex); + String rest = response.substring(includeIndex + "@include(".length()); + int filenameEnd = rest.indexOf(")"); + String includeFileName = rest.substring(0, filenameEnd); + String includedContent = IOUtils.readFile(new File(responsesPath + includeFileName)); + includedContent = include(includedContent); + String postFix = rest.substring(filenameEnd + 1); + postFix = include(postFix); + return prefix + includedContent + postFix; + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-config-server.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-config-server.json index 55891309856..55891309856 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-config-server.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-config-server.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-tenant-node.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-tenant-node.json index 16cf5c36dcc..16cf5c36dcc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/acl-tenant-node.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/acl-tenant-node.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json index c61a755d989..c61a755d989 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/active-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/active-nodes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json new file mode 100644 index 00000000000..ce2309e4d6c --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json @@ -0,0 +1,56 @@ +{ + "url" : "http://localhost:8080/nodes/v2/applications/tenant1.application1.instance1", + "id" : "tenant1.application1.instance1", + "clusters" : { + "id1" : { + "min" : { + "nodes" : 2, + "groups" : 1, + "resources" : { + "vcpu" : 2.0, + "memoryGb" : 8.0, + "diskGb" : 50.0, + "bandwidthGbps" : 1.0, + "diskSpeed" : "fast", + "storageType" : "any" + } + }, + "max" : { + "nodes" : 8, + "groups" : 2, + "resources" : { + "vcpu" : 4.0, + "memoryGb" : 16.0, + "diskGb" : 1000.0, + "bandwidthGbps" : 1.0, + "diskSpeed" : "fast", + "storageType" : "any" + } + }, + "suggested" : { + "nodes" : 6, + "groups" : 2, + "resources" : { + "vcpu" : 3.0, + "memoryGb" : 20.0, + "diskGb" : 100.0, + "bandwidthGbps" : 1.0, + "diskSpeed" : "fast", + "storageType" : "any" + } + }, + "target" : { + "nodes" : 4, + "groups" : 1, + "resources" : { + "vcpu" : 3.0, + "memoryGb" : 16.0, + "diskGb" : 100.0, + "bandwidthGbps" : 1.0, + "diskSpeed" : "fast", + "storageType" : "any" + } + } + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2-nodes.json index 4581ecba73d..4581ecba73d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/application2-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2-nodes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json new file mode 100644 index 00000000000..75af5d4a328 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json @@ -0,0 +1,32 @@ +{ + "url": "http://localhost:8080/nodes/v2/applications/tenant2.application2.instance2", + "id": "tenant2.application2.instance2", + "clusters": { + "id2": { + "min": { + "nodes": 2, + "groups": 1, + "resources": { + "vcpu": 2.0, + "memoryGb": 8.0, + "diskGb": 50.0, + "bandwidthGbps": 1.0, + "diskSpeed": "fast", + "storageType": "any" + } + }, + "max": { + "nodes": 2, + "groups": 1, + "resources": { + "vcpu": 2.0, + "memoryGb": 8.0, + "diskGb": 50.0, + "bandwidthGbps": 1.0, + "diskSpeed": "fast", + "storageType": "any" + } + } + } + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/applications.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/applications.json new file mode 100644 index 00000000000..6d25b3e59f5 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/applications.json @@ -0,0 +1,20 @@ +{ + "applications" : [ + { + "url" : "http://localhost:8080/nodes/v2/applications/tenant4.application4.instance4", + "id" : "tenant4.application4.instance4" + }, + { + "url" : "http://localhost:8080/nodes/v2/applications/tenant2.application2.instance2", + "id" : "tenant2.application2.instance2" + }, + { + "url" : "http://localhost:8080/nodes/v2/applications/tenant3.application3.instance3", + "id" : "tenant3.application3.instance3" + }, + { + "url" : "http://localhost:8080/nodes/v2/applications/tenant1.application1.instance1", + "id" : "tenant1.application1.instance1" + } + ] +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-impossible.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-impossible.json index f85e0dcc270..f85e0dcc270 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-impossible.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-impossible.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-possible.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-possible.json index 177e1f317f7..177e1f317f7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-hostremoval-possible.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-hostremoval-possible.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-zone.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-zone.json index f6b2f96023a..f6b2f96023a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/capacity-zone.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/capacity-zone.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json index d2d39b06161..d2d39b06161 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json index 969c9a1b9c2..969c9a1b9c2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/cfg2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/child-nodes.json index dae92aae091..dae92aae091 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/child-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/child-nodes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/content-nodes.json index e84f17dc40d..e84f17dc40d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/content-nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/content-nodes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/controller1.json index 035e15ead49..035e15ead49 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/controller1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/controller1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json index c97509b3b3d..c97509b3b3d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-container1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json index 77622933866..77622933866 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade-complete.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade-complete.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade.json index a1775dc794e..a1775dc794e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1-os-upgrade.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1-os-upgrade.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1.json index b6b93c8d108..b6b93c8d108 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json index 88e4d132ed5..88e4d132ed5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node3.json index c890f1e72be..c890f1e72be 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node3.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node4.json index 197a8c248e9..197a8c248e9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node4.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node5.json index 2638b6b74f5..2638b6b74f5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/docker-node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node5.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json index ea5c3623303..ea5c3623303 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/dockerhost1-with-firmware-data.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/dockerhost1-with-firmware-data.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags1.json index a606777e9fd..a606777e9fd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags2.json index 4baf75f2169..4baf75f2169 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/flags2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/flags2.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers-single.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json index a9a728bab15..a9a728bab15 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers-single.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json index 515081bcb8e..515081bcb8e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/load-balancers.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json index ab608bac2b4..e041a7b8b54 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/maintenance.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/maintenance.json @@ -53,6 +53,9 @@ }, { "name": "RetiredExpirer" + }, + { + "name":"ScalingSuggestionsMaintainer" } ], "inactive": [ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json index 7f6a6e17fa1..7f6a6e17fa1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json index da0e7b28f3e..da0e7b28f3e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node10.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json index 17f8210fa4d..17f8210fa4d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node11.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json index 2e96c4c9bbc..2e96c4c9bbc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node13.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json index bfbecc09c60..bfbecc09c60 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node14.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json index b8ce5c86c4d..b8ce5c86c4d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json index 70ae2a030bd..70ae2a030bd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node3.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json index af3552945d9..af3552945d9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json index c8680e1b420..c8680e1b420 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json index beb42ec6907..beb42ec6907 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5-after-changes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json index 3fb5ffa4708..3fb5ffa4708 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node5.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node5.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json index 3107311b792..3107311b792 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node55.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node55.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-after-changes.json index 65c7e9db6cd..65c7e9db6cd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-after-changes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json index a3d53798d7c..a3d53798d7c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-2.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json index 7f0c3a5f706..7f0c3a5f706 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports-3.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports-3.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json index 67b8d67c7f1..67b8d67c7f1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6-reports.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6-reports.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json index 5a665a5a223..5a665a5a223 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json index 38757b81156..38757b81156 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node7.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node7.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node8.json index aa2b2acdb9f..aa2b2acdb9f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node8.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node8.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json index 5f388c13f28..5f388c13f28 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/node9.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node9.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json index cf3e3c1b457..cf3e3c1b457 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes-recursive.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json index cdbd2cac9b9..cdbd2cac9b9 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/nodes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/nodes.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent1.json index 40f4a3b5160..40f4a3b5160 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent1.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json index ecc497172c7..ecc497172c7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/parent2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/parent2.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/root.json index 86becefb146..cc91314df84 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/root.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/root.json @@ -14,6 +14,9 @@ }, { "url":"http://localhost:8080/nodes/v2/upgrade/" + }, + { + "url":"http://localhost:8080/nodes/v2/application/" } ] }
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json index f8343756559..f8343756559 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states-recursive.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states-recursive.json diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states.json index 69579148df3..69579148df3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/responses/states.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/states.json diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java index b1f205f69d8..fbb25baa806 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/OrchestratorContext.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.time.TimeBudget; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.orchestrator.controller.ClusterControllerClientTimeouts; @@ -153,7 +153,7 @@ public class OrchestratorContext implements AutoCloseable { try { unlock.run(); } catch (RuntimeException e) { - logger.log(LogLevel.ERROR, "Failed run on close : " + e.getMessage()); + logger.log(Level.SEVERE, "Failed run on close : " + e.getMessage()); } }); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java index 9900c8de752..72033e41192 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/model/StorageNodeImpl.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.orchestrator.model; import com.google.common.util.concurrent.UncheckedTimeoutException; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; @@ -62,7 +62,7 @@ public class StorageNodeImpl implements StorageNode { ConfigId configId = storageService.configId(); int nodeIndex = VespaModelUtil.getStorageNodeIndex(configId); - logger.log(LogLevel.DEBUG, () -> "Setting cluster controller state for " + + logger.log(Level.FINE, () -> "Setting cluster controller state for " + "application " + applicationInstance.reference().asString() + ", host " + hostName() + ", cluster name " + clusterId + diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java index 58edbdc4218..6a118f9d606 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/ApplicationSuspensionResource.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.orchestrator.resources; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.orchestrator.ApplicationIdNotFoundException; import com.yahoo.vespa.orchestrator.ApplicationStateChangeDeniedException; import com.yahoo.vespa.orchestrator.OrchestratorImpl; @@ -83,13 +83,13 @@ public class ApplicationSuspensionResource implements ApplicationSuspensionApi { try { orchestrator.suspend(applicationId); } catch (ApplicationIdNotFoundException e) { - log.log(LogLevel.INFO, "ApplicationId " + applicationIdString + " not found.", e); + log.log(Level.INFO, "ApplicationId " + applicationIdString + " not found.", e); throw new NotFoundException(e); } catch (ApplicationStateChangeDeniedException e) { - log.log(LogLevel.INFO, "Suspend for " + applicationIdString + " failed.", e); + log.log(Level.INFO, "Suspend for " + applicationIdString + " failed.", e); throw new WebApplicationException(Response.Status.CONFLICT); } catch (RuntimeException e) { - log.log(LogLevel.INFO, "Suspend for " + applicationIdString + " failed from unknown reasons", e); + log.log(Level.INFO, "Suspend for " + applicationIdString + " failed from unknown reasons", e); throw new InternalServerErrorException(e); } } @@ -100,13 +100,13 @@ public class ApplicationSuspensionResource implements ApplicationSuspensionApi { try { orchestrator.resume(applicationId); } catch (ApplicationIdNotFoundException e) { - log.log(LogLevel.INFO, "ApplicationId " + applicationIdString + " not found.", e); + log.log(Level.INFO, "ApplicationId " + applicationIdString + " not found.", e); throw new NotFoundException(e); } catch (ApplicationStateChangeDeniedException e) { - log.log(LogLevel.INFO, "Suspend for " + applicationIdString + " failed.", e); + log.log(Level.INFO, "Suspend for " + applicationIdString + " failed.", e); throw new WebApplicationException(Response.Status.CONFLICT); } catch (RuntimeException e) { - log.log(LogLevel.INFO, "Suspend for " + applicationIdString + " failed from unknown reasons", e); + log.log(Level.INFO, "Suspend for " + applicationIdString + " failed from unknown reasons", e); throw new InternalServerErrorException(e); } } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java index 054d3320215..9ed9e1b9063 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.orchestrator.resources; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.Host; import com.yahoo.vespa.orchestrator.HostNameNotFoundException; @@ -78,10 +78,10 @@ public class HostResource implements HostApi { applicationUri.toString(), hostServices); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.DEBUG, "Failed to get host " + hostName + ": " + e.getMessage()); + log.log(Level.FINE, "Failed to get host " + hostName + ": " + e.getMessage()); throw webExceptionFromTimeout("getHost", hostName, e); } catch (HostNameNotFoundException e) { - log.log(LogLevel.DEBUG, "Host not found: " + hostName); + log.log(Level.FINE, "Host not found: " + hostName); throw new NotFoundException(e); } } @@ -101,14 +101,14 @@ public class HostResource implements HostApi { try { orchestrator.setNodeStatus(hostName, state); } catch (HostNameNotFoundException e) { - log.log(LogLevel.DEBUG, "Host not found: " + hostName); + log.log(Level.FINE, "Host not found: " + hostName); throw new NotFoundException(e); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.DEBUG, "Failed to patch " + hostName + ": " + e.getMessage()); + log.log(Level.FINE, "Failed to patch " + hostName + ": " + e.getMessage()); throw webExceptionFromTimeout("patch", hostName, e); } catch (OrchestrationException e) { String message = "Failed to set " + hostName + " to " + state + ": " + e.getMessage(); - log.log(LogLevel.DEBUG, message, e); + log.log(Level.FINE, message, e); throw new InternalServerErrorException(message); } } @@ -124,13 +124,13 @@ public class HostResource implements HostApi { try { orchestrator.suspend(hostName); } catch (HostNameNotFoundException e) { - log.log(LogLevel.DEBUG, "Host not found: " + hostName); + log.log(Level.FINE, "Host not found: " + hostName); throw new NotFoundException(e); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.DEBUG, "Failed to suspend " + hostName + ": " + e.getMessage()); + log.log(Level.FINE, "Failed to suspend " + hostName + ": " + e.getMessage()); throw webExceptionFromTimeout("suspend", hostName, e); } catch (HostStateChangeDeniedException e) { - log.log(LogLevel.DEBUG, "Failed to suspend " + hostName + ": " + e.getMessage()); + log.log(Level.FINE, "Failed to suspend " + hostName + ": " + e.getMessage()); throw webExceptionWithDenialReason("suspend", hostName, e); } return new UpdateHostResponse(hostName.s(), null); @@ -142,13 +142,13 @@ public class HostResource implements HostApi { try { orchestrator.resume(hostName); } catch (HostNameNotFoundException e) { - log.log(LogLevel.DEBUG, "Host not found: " + hostName); + log.log(Level.FINE, "Host not found: " + hostName); throw new NotFoundException(e); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.DEBUG, "Failed to resume " + hostName + ": " + e.getMessage()); + log.log(Level.FINE, "Failed to resume " + hostName + ": " + e.getMessage()); throw webExceptionFromTimeout("resume", hostName, e); } catch (HostStateChangeDeniedException e) { - log.log(LogLevel.DEBUG, "Failed to resume " + hostName + ": " + e.getMessage()); + log.log(Level.FINE, "Failed to resume " + hostName + ": " + e.getMessage()); throw webExceptionWithDenialReason("resume", hostName, e); } return new UpdateHostResponse(hostName.s(), null); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java index 6e857563f9b..3d60d40dfcf 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostSuspensionResource.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.orchestrator.resources; import com.google.common.util.concurrent.UncheckedTimeoutException; import com.yahoo.container.jaxrs.annotation.Component; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.orchestrator.BatchHostNameNotFoundException; import com.yahoo.vespa.orchestrator.BatchInternalErrorException; @@ -43,21 +43,21 @@ public class HostSuspensionResource implements HostSuspensionApi { try { orchestrator.suspendAll(parentHostname, hostnames); } catch (BatchHostStateChangeDeniedException e) { - log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); + log.log(Level.FINE, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT); } catch (UncheckedTimeoutException e) { - log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); + log.log(Level.FINE, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); throw createWebApplicationException(e.getMessage(), Response.Status.CONFLICT); } catch (BatchHostNameNotFoundException e) { - log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); + log.log(Level.FINE, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); // Note that we're returning BAD_REQUEST instead of NOT_FOUND because the resource identified // by the URL path was found. It's one of the hostnames in the request it failed to find. throw createWebApplicationException(e.getMessage(), Response.Status.BAD_REQUEST); } catch (BatchInternalErrorException e) { - log.log(LogLevel.DEBUG, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); + log.log(Level.FINE, "Failed to suspend nodes " + hostnames + " with parent host " + parentHostname, e); throw createWebApplicationException(e.getMessage(), Response.Status.INTERNAL_SERVER_ERROR); } - log.log(LogLevel.DEBUG, "Suspended " + hostnames + " with parent " + parentHostname); + log.log(Level.FINE, "Suspended " + hostnames + " with parent " + parentHostname); return BatchOperationResult.successResult(); } diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java index 99f1f00e7dc..cbb6902066e 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/HostInfosServiceImpl.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.orchestrator.status; import com.yahoo.config.provision.ApplicationId; import com.yahoo.jdisc.Timer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; @@ -109,7 +109,7 @@ public class HostInfosServiceImpl implements HostInfosService { curator.framework().delete().forPath(path); return true; } catch (NoNodeException e) { - log.log(LogLevel.DEBUG, debugLogMessageIfNotExists, e); + log.log(Level.FINE, debugLogMessageIfNotExists, e); return false; } catch (Exception e) { throw new RuntimeException(e); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java index dfaaecbcf82..1a8105e682f 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkApplicationLock.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.orchestrator.status; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.curator.Curator; @@ -82,7 +82,7 @@ class ZkApplicationLock implements ApplicationLock { onClose.run(); } catch (RuntimeException e) { // We may want to avoid logging some exceptions that may be expected, like when session expires. - log.log(LogLevel.WARNING, + log.log(Level.WARNING, "Failed close application lock in " + ZkApplicationLock.class.getSimpleName() + ", will ignore and continue", e); diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java index 246a70992c7..32fdaa9433d 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/status/ZkStatusService.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jaxrs.annotation.Component; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Timer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; @@ -93,7 +93,7 @@ public class ZkStatusService implements StatusService { return resultSet; } catch (Exception e) { - log.log(LogLevel.DEBUG, "Something went wrong while listing out applications in suspend.", e); + log.log(Level.FINE, "Something went wrong while listing out applications in suspend.", e); throw new RuntimeException(e); } } @@ -195,7 +195,7 @@ public class ZkStatusService implements StatusService { lock.close(); } catch (RuntimeException e) { // We may want to avoid logging some exceptions that may be expected, like when session expires. - log.log(LogLevel.WARNING, + log.log(Level.WARNING, "Failed to close application lock for " + ZkStatusService.class.getSimpleName() + ", will ignore and continue", e); @@ -277,7 +277,7 @@ public class ZkStatusService implements StatusService { try { lock = lockApplication(context, reference); } catch (RuntimeException e) { - log.log(LogLevel.ERROR, "Failed to get Orchestrator lock on when " + reference + + log.log(Level.SEVERE, "Failed to get Orchestrator lock on when " + reference + eventDescription + ": " + e.getMessage()); return; } @@ -285,7 +285,7 @@ public class ZkStatusService implements StatusService { try (lock) { runnable.run(); } catch (RuntimeException e) { - log.log(LogLevel.ERROR, "Failed to clean up after " + reference + eventDescription + + log.log(Level.SEVERE, "Failed to clean up after " + reference + eventDescription + ": " + e.getMessage()); } } diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java index ddff782cdb1..02c336e4109 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/status/ZkStatusServiceTest.java @@ -6,7 +6,7 @@ import com.yahoo.exception.ExceptionUtils; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.Timer; import com.yahoo.jdisc.test.TestTimer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.test.ManualClock; import com.yahoo.vespa.applicationmodel.ApplicationInstanceReference; import com.yahoo.vespa.applicationmodel.HostName; @@ -82,7 +82,7 @@ public class ZkStatusServiceTest { @Before public void setUp() throws Exception { - Logger.getLogger("").setLevel(LogLevel.WARNING); + Logger.getLogger("").setLevel(Level.WARNING); testingServer = new TestingServer(); curator = createConnectedCurator(testingServer); diff --git a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp index 5c8b5b029d2..298e29cbecb 100644 --- a/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp +++ b/persistence/src/vespa/persistence/conformancetest/conformancetest.cpp @@ -300,8 +300,6 @@ feedDocs(PersistenceProvider& spi, EXPECT_TRUE(!result.hasError()); docs.push_back(DocAndTimestamp(doc, Timestamp(1000 + i))); } - spi.flush(bucket, context); - EXPECT_EQ(Result(), Result(spi.flush(bucket, context))); return docs; } @@ -354,8 +352,6 @@ TEST_F(ConformanceTest, testBasics) Result(), Result(spi->remove(bucket, Timestamp(3), doc1->getId(), context))); - EXPECT_EQ(Result(), Result(spi->flush(bucket, context))); - // Iterate first without removes, then with. for (int iterPass = 0; iterPass < 2; ++iterPass) { bool includeRemoves = (iterPass == 1); @@ -441,11 +437,8 @@ TEST_F(ConformanceTest, testListBuckets) spi->createBucket(bucket3, context); spi->put(bucket1, Timestamp(1), doc1, context); - spi->flush(bucket1, context); spi->put(bucket2, Timestamp(2), doc2, context); - spi->flush(bucket2, context); spi->put(bucket3, Timestamp(3), doc3, context); - spi->flush(bucket3, context); { BucketIdListResult result = spi->listBuckets(makeBucketSpace(), PartitionId(1)); @@ -482,7 +475,6 @@ TEST_F(ConformanceTest, testBucketInfo) spi->put(bucket, Timestamp(2), doc2, context); const BucketInfo info1 = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); { EXPECT_EQ(1, (int)info1.getDocumentCount()); @@ -492,7 +484,6 @@ TEST_F(ConformanceTest, testBucketInfo) spi->put(bucket, Timestamp(3), doc1, context); const BucketInfo info2 = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); { EXPECT_EQ(2, (int)info2.getDocumentCount()); @@ -503,7 +494,6 @@ TEST_F(ConformanceTest, testBucketInfo) spi->put(bucket, Timestamp(4), doc1, context); const BucketInfo info3 = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); { EXPECT_EQ(2, (int)info3.getDocumentCount()); @@ -514,7 +504,6 @@ TEST_F(ConformanceTest, testBucketInfo) spi->remove(bucket, Timestamp(5), doc1->getId(), context); const BucketInfo info4 = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); { EXPECT_EQ(1, (int)info4.getDocumentCount()); @@ -544,7 +533,6 @@ TEST_F(ConformanceTest, testOrderIndependentBucketInfo) { spi->put(bucket, Timestamp(2), doc1, context); spi->put(bucket, Timestamp(3), doc2, context); - spi->flush(bucket, context); const BucketInfo info(spi->getBucketInfo(bucket).getBucketInfo()); checksumOrdered = info.getChecksum(); @@ -563,7 +551,6 @@ TEST_F(ConformanceTest, testOrderIndependentBucketInfo) // Swap order of puts spi->put(bucket, Timestamp(3), doc2, context); spi->put(bucket, Timestamp(2), doc1, context); - spi->flush(bucket, context); const BucketInfo info(spi->getBucketInfo(bucket).getBucketInfo()); checksumUnordered = info.getChecksum(); @@ -589,7 +576,6 @@ TEST_F(ConformanceTest, testPut) { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info.getDocumentCount()); EXPECT_TRUE(info.getEntryCount() >= info.getDocumentCount()); @@ -615,7 +601,6 @@ TEST_F(ConformanceTest, testPutNewDocumentVersion) Result result = spi->put(bucket, Timestamp(3), doc1, context); { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info.getDocumentCount()); EXPECT_TRUE(info.getEntryCount() >= info.getDocumentCount()); @@ -627,7 +612,6 @@ TEST_F(ConformanceTest, testPutNewDocumentVersion) result = spi->put(bucket, Timestamp(4), doc2, context); { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info.getDocumentCount()); EXPECT_TRUE(info.getEntryCount() >= info.getDocumentCount()); @@ -666,7 +650,6 @@ TEST_F(ConformanceTest, testPutOlderDocumentVersion) Result result = spi->put(bucket, Timestamp(5), doc1, context); const BucketInfo info1 = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); { EXPECT_EQ(1, (int)info1.getDocumentCount()); EXPECT_TRUE(info1.getEntryCount() >= info1.getDocumentCount()); @@ -678,7 +661,6 @@ TEST_F(ConformanceTest, testPutOlderDocumentVersion) result = spi->put(bucket, Timestamp(4), doc2, context); { const BucketInfo info2 = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info2.getDocumentCount()); EXPECT_TRUE(info2.getEntryCount() >= info1.getDocumentCount()); @@ -712,7 +694,6 @@ TEST_F(ConformanceTest, testPutDuplicate) BucketChecksum checksum; { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info.getDocumentCount()); checksum = info.getChecksum(); } @@ -721,7 +702,6 @@ TEST_F(ConformanceTest, testPutDuplicate) { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info.getDocumentCount()); EXPECT_EQ(checksum, info.getChecksum()); } @@ -746,7 +726,6 @@ TEST_F(ConformanceTest, testRemove) { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(1, (int)info.getDocumentCount()); EXPECT_TRUE(info.getChecksum() != 0); @@ -764,7 +743,6 @@ TEST_F(ConformanceTest, testRemove) { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(0, (int)info.getDocumentCount()); EXPECT_EQ(0, (int)info.getChecksum()); @@ -791,7 +769,6 @@ TEST_F(ConformanceTest, testRemove) context); { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(0, (int)info.getDocumentCount()); EXPECT_EQ(0, (int)info.getChecksum()); @@ -799,7 +776,6 @@ TEST_F(ConformanceTest, testRemove) } Result result4 = spi->put(bucket, Timestamp(9), doc1, context); - spi->flush(bucket, context); EXPECT_TRUE(!result4.hasError()); @@ -809,7 +785,6 @@ TEST_F(ConformanceTest, testRemove) context); { const BucketInfo info = spi->getBucketInfo(bucket).getBucketInfo(); - spi->flush(bucket, context); EXPECT_EQ(0, (int)info.getDocumentCount()); EXPECT_EQ(0, (int)info.getChecksum()); @@ -847,7 +822,6 @@ TEST_F(ConformanceTest, testRemoveMerge) Timestamp(10), removeId, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, removeResult.getErrorCode()); EXPECT_EQ(false, removeResult.wasFound()); } @@ -875,7 +849,6 @@ TEST_F(ConformanceTest, testRemoveMerge) Timestamp(11), removeId, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, removeResult.getErrorCode()); EXPECT_EQ(false, removeResult.wasFound()); } @@ -903,7 +876,6 @@ TEST_F(ConformanceTest, testRemoveMerge) Timestamp(7), removeId, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, removeResult.getErrorCode()); EXPECT_EQ(false, removeResult.wasFound()); } @@ -946,7 +918,6 @@ TEST_F(ConformanceTest, testUpdate) { UpdateResult result = spi->update(bucket, Timestamp(3), update, context); - spi->flush(bucket, context); EXPECT_EQ(Result(), Result(result)); EXPECT_EQ(Timestamp(0), result.getExistingTimestamp()); } @@ -955,7 +926,6 @@ TEST_F(ConformanceTest, testUpdate) { UpdateResult result = spi->update(bucket, Timestamp(4), update, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); EXPECT_EQ(Timestamp(3), result.getExistingTimestamp()); @@ -975,7 +945,6 @@ TEST_F(ConformanceTest, testUpdate) } spi->remove(bucket, Timestamp(5), doc1->getId(), context); - spi->flush(bucket, context); { GetResult result = spi->get(bucket, @@ -991,7 +960,6 @@ TEST_F(ConformanceTest, testUpdate) { UpdateResult result = spi->update(bucket, Timestamp(6), update, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); EXPECT_EQ(Timestamp(0), result.getExistingTimestamp()); @@ -1009,7 +977,6 @@ TEST_F(ConformanceTest, testUpdate) // Document does not exist (and therefore its condition cannot match by definition), // but since CreateIfNonExistent is set it should be auto-created anyway. UpdateResult result = spi->update(bucket, Timestamp(7), update, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, result.getErrorCode()); EXPECT_EQ(Timestamp(7), result.getExistingTimestamp()); } @@ -1044,7 +1011,6 @@ TEST_F(ConformanceTest, testGet) } spi->put(bucket, Timestamp(3), doc1, context); - spi->flush(bucket, context); { GetResult result = spi->get(bucket, document::AllFields(), @@ -1054,7 +1020,6 @@ TEST_F(ConformanceTest, testGet) } spi->remove(bucket, Timestamp(4), doc1->getId(), context); - spi->flush(bucket, context); { GetResult result = spi->get(bucket, document::AllFields(), @@ -1168,7 +1133,6 @@ TEST_F(ConformanceTest, testIterateAllDocsNewestVersionOnly) spi->put(b, newTimestamp, newDoc, context); newDocs.push_back(DocAndTimestamp(newDoc, newTimestamp)); } - spi->flush(b, context); CreateIteratorResult iter(createIterator(*spi, b, createSelection(""))); @@ -1249,7 +1213,6 @@ TEST_F(ConformanceTest, testIterateMatchTimestampRange) DocAndTimestamp(doc, Timestamp(1000 + i))); } } - spi->flush(b, context); Selection sel = Selection(DocumentSelection("")); sel.setFromTimestamp(fromTimestamp); @@ -1295,7 +1258,6 @@ TEST_F(ConformanceTest, testIterateExplicitTimestampSubset) Timestamp(2000), docsToVisit.front().doc->getId(), context) .wasFound()); - spi->flush(b, context); timestampsToVisit.push_back(Timestamp(2000)); removes.insert(docsToVisit.front().doc->getId().toString()); @@ -1339,7 +1301,6 @@ TEST_F(ConformanceTest, testIterateRemoves) nonRemovedDocs.push_back(docs[i]); } } - spi->flush(b, context); // First, test iteration without removes { @@ -1387,7 +1348,6 @@ TEST_F(ConformanceTest, testIterateMatchSelection) DocAndTimestamp(doc, Timestamp(1000 + i))); } } - spi->flush(b, context); CreateIteratorResult iter( createIterator(*spi, @@ -1416,7 +1376,6 @@ TEST_F(ConformanceTest, testIterationRequiringDocumentIdOnlyMatching) // remove entry for it regardless. EXPECT_TRUE( !spi->remove(b, Timestamp(2000), removedId, context).wasFound()); - spi->flush(b, context); Selection sel(createSelection("id == '" + removedId.toString() + "'")); @@ -1530,7 +1489,6 @@ TEST_F(ConformanceTest, testDeleteBucket) spi->createBucket(bucket, context); spi->put(bucket, Timestamp(3), doc1, context); - spi->flush(bucket, context); spi->deleteBucket(bucket, context); testDeleteBucketPostCondition(spi, bucket, *doc1); @@ -1586,8 +1544,6 @@ TEST_F(ConformanceTest, testSplitNormalCase) spi->put(bucketC, Timestamp(i + 1), doc1, context); } - spi->flush(bucketC, context); - spi->split(bucketC, bucketA, bucketB, context); testSplitNormalCasePostCondition(spi, bucketA, bucketB, bucketC, testDocMan); @@ -1657,28 +1613,23 @@ TEST_F(ConformanceTest, testSplitTargetExists) spi->put(bucketC, Timestamp(i + 1), doc1, context); } - spi->flush(bucketC, context); for (uint32_t i = 10; i < 20; ++i) { Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); spi->put(bucketB, Timestamp(i + 1), doc1, context); } - spi->flush(bucketB, context); EXPECT_TRUE(!spi->getBucketInfo(bucketB).getBucketInfo().isActive()); for (uint32_t i = 10; i < 20; ++i) { Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); spi->put(bucketC, Timestamp(i + 1), doc1, context); } - spi->flush(bucketC, context); for (uint32_t i = 20; i < 25; ++i) { Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); spi->put(bucketB, Timestamp(i + 1), doc1, context); } - spi->flush(bucketB, context); - spi->split(bucketC, bucketA, bucketB, context); testSplitTargetExistsPostCondition(spi, bucketA, bucketB, bucketC, testDocMan); @@ -1745,8 +1696,6 @@ TEST_F(ConformanceTest, testSplitSingleDocumentInSource) Document::SP doc = testDocMan.createRandomDocumentAtLocation(0x06, 0); spi->put(source, Timestamp(1), doc, context); - spi->flush(source, context); - spi->split(source, target1, target2, context); testSplitSingleDocumentInSourcePostCondition( spi, source, target1, target2, testDocMan); @@ -1804,7 +1753,6 @@ ConformanceTest::createAndPopulateJoinSourceBuckets( source1.getBucketId().getId(), i)); spi.put(source1, Timestamp(i + 1), doc, context); } - spi.flush(source1, context); for (uint32_t i = 10; i < 20; ++i) { Document::SP doc( @@ -1812,7 +1760,6 @@ ConformanceTest::createAndPopulateJoinSourceBuckets( source2.getBucketId().getId(), i)); spi.put(source2, Timestamp(i + 1), doc, context); } - spi.flush(source2, context); } void @@ -1912,21 +1859,17 @@ TEST_F(ConformanceTest, testJoinTargetExists) spi->put(bucketA, Timestamp(i + 1), doc1, context); } - spi->flush(bucketA, context); for (uint32_t i = 10; i < 20; ++i) { Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); spi->put(bucketB, Timestamp(i + 1), doc1, context); } - spi->flush(bucketB, context); for (uint32_t i = 20; i < 30; ++i) { Document::SP doc1 = testDocMan.createRandomDocumentAtLocation(0x06, i); spi->put(bucketC, Timestamp(i + 1), doc1, context); } - spi->flush(bucketC, context); - spi->join(bucketA, bucketB, bucketC, context); testJoinTargetExistsPostCondition(spi, bucketA, bucketB, bucketC, testDocMan); @@ -1991,7 +1934,6 @@ ConformanceTest::populateBucket(const Bucket& b, location, i); spi.put(b, Timestamp(i + 1), doc1, context); } - spi.flush(b, context); } TEST_F(ConformanceTest, testJoinOneBucket) @@ -2150,7 +2092,6 @@ TEST_F(ConformanceTest, testMaintain) spi->createBucket(bucket, context); spi->put(bucket, Timestamp(3), doc1, context); - spi->flush(bucket, context); EXPECT_EQ(Result::ErrorType::NONE, spi->maintain(bucket, LOW).getErrorCode()); @@ -2229,7 +2170,6 @@ TEST_F(SingleDocTypeConformanceTest, testBucketActivationSplitAndJoin) spi->createBucket(bucketC, context); spi->put(bucketC, Timestamp(1), doc1, context); spi->put(bucketC, Timestamp(2), doc2, context); - spi->flush(bucketC, context); spi->setActiveState(bucketC, BucketInfo::ACTIVE); EXPECT_TRUE(spi->getBucketInfo(bucketC).getBucketInfo().isActive()); @@ -2304,14 +2244,11 @@ TEST_F(ConformanceTest, testRemoveEntry) spi->createBucket(bucket, context); spi->put(bucket, Timestamp(3), doc1, context); - spi->flush(bucket, context); BucketInfo info1 = spi->getBucketInfo(bucket).getBucketInfo(); { spi->put(bucket, Timestamp(4), doc2, context); - spi->flush(bucket, context); spi->removeEntry(bucket, Timestamp(4), context); - spi->flush(bucket, context); BucketInfo info2 = spi->getBucketInfo(bucket).getBucketInfo(); EXPECT_EQ(info1, info2); } @@ -2319,9 +2256,7 @@ TEST_F(ConformanceTest, testRemoveEntry) // Test case where there exists a previous version of the document. { spi->put(bucket, Timestamp(5), doc1, context); - spi->flush(bucket, context); spi->removeEntry(bucket, Timestamp(5), context); - spi->flush(bucket, context); BucketInfo info2 = spi->getBucketInfo(bucket).getBucketInfo(); EXPECT_EQ(info1, info2); } @@ -2329,14 +2264,11 @@ TEST_F(ConformanceTest, testRemoveEntry) // Test case where the newest document version after removeEntrying is a remove. { spi->remove(bucket, Timestamp(6), doc1->getId(), context); - spi->flush(bucket, context); BucketInfo info2 = spi->getBucketInfo(bucket).getBucketInfo(); EXPECT_EQ(uint32_t(0), info2.getDocumentCount()); spi->put(bucket, Timestamp(7), doc1, context); - spi->flush(bucket, context); spi->removeEntry(bucket, Timestamp(7), context); - spi->flush(bucket, context); BucketInfo info3 = spi->getBucketInfo(bucket).getBucketInfo(); EXPECT_EQ(info2, info3); } @@ -2395,9 +2327,6 @@ TEST_F(ConformanceTest, testBucketSpaces) spi->put(bucket01, Timestamp(4), doc2, context); spi->put(bucket11, Timestamp(5), doc3, context); spi->put(bucket12, Timestamp(6), doc4, context); - spi->flush(bucket01, context); - spi->flush(bucket11, context); - spi->flush(bucket12, context); // Check bucket lists assertBucketList(*spi, bucketSpace0, partId, { bucketId1 }); assertBucketList(*spi, bucketSpace1, partId, { bucketId1, bucketId2 }); diff --git a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h index 557a9ec2edd..cbd76951004 100644 --- a/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/abstractpersistenceprovider.h @@ -41,11 +41,6 @@ public: Result removeEntry(const Bucket&, Timestamp, Context&) override { return Result(); } /** - * Default impl is getBucketInfo(); - */ - Result flush(const Bucket&, Context&) override { return Result(); } - - /** * Default impl is remove(). */ RemoveResult removeIfFound(const Bucket&, Timestamp, const DocumentId&, Context&) override; diff --git a/persistence/src/vespa/persistence/spi/persistenceprovider.h b/persistence/src/vespa/persistence/spi/persistenceprovider.h index 857630e2606..c70d5e3f1c3 100644 --- a/persistence/src/vespa/persistence/spi/persistenceprovider.h +++ b/persistence/src/vespa/persistence/spi/persistenceprovider.h @@ -202,30 +202,6 @@ struct PersistenceProvider virtual UpdateResult update(const Bucket&, Timestamp timestamp, const DocumentUpdateSP& update, Context&) = 0; /** - * The service layer may choose to batch certain commands. This means that - * the service layer will lock the bucket only once, then perform several - * commands, and finally get the bucket info from the bucket, and then - * flush it. This can be used to improve performance by caching the - * modifications, and persisting them to disk only when flush is called. - * The service layer guarantees that after one of these operations, flush() - * is called, regardless of whether the operation succeeded or not, before - * another bucket is processed in the same worker thead. The following - * operations can be batched and have the guarantees - * above: - * - put - * - get - * - remove (all versions) - * - update - * - revert - * - join - * <p/> - * A provider may of course choose to not sync to disk at flush time either, - * but then data may be more prone to being lost on node issues, and the - * provider must figure out when to flush its cache itself. - */ - virtual Result flush(const Bucket&, Context&) = 0; - - /** * Retrieves the latest version of the document specified by the * document id. If no versions were found, or the document was removed, * the result should be successful, but contain no document (see GetResult). diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchcommon/src/vespa/searchcommon/attribute/config.h index 836fcfed84a..df5aa9e217a 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.h +++ b/searchcommon/src/vespa/searchcommon/attribute/config.h @@ -71,6 +71,10 @@ public: _hnsw_index_params = params; return *this; } + Config& clear_hnsw_index_params() { + _hnsw_index_params.reset(); + return *this; + } /** * Enable attribute posting list to consist of a bitvector in diff --git a/searchcore/src/apps/proton/downpersistence.cpp b/searchcore/src/apps/proton/downpersistence.cpp index 4b911eb6d2b..d4ec9cee395 100644 --- a/searchcore/src/apps/proton/downpersistence.cpp +++ b/searchcore/src/apps/proton/downpersistence.cpp @@ -98,12 +98,6 @@ UpdateResult DownPersistence::update(const Bucket&, Timestamp, errorResult.getErrorMessage()); } -Result -DownPersistence::flush(const Bucket&, Context&) -{ - return errorResult; -} - GetResult DownPersistence::get(const Bucket&, const document::FieldSet&, const DocumentId&, Context&) const diff --git a/searchcore/src/apps/proton/downpersistence.h b/searchcore/src/apps/proton/downpersistence.h index 0a602c4467e..8cdac7aaa1b 100644 --- a/searchcore/src/apps/proton/downpersistence.h +++ b/searchcore/src/apps/proton/downpersistence.h @@ -37,7 +37,6 @@ public: RemoveResult removeIfFound(const Bucket&, Timestamp timestamp, const DocumentId& id, Context&) override; Result removeEntry(const Bucket&, Timestamp, Context&) override; UpdateResult update(const Bucket&, Timestamp timestamp, const DocumentUpdateSP& update, Context&) override; - Result flush(const Bucket&, Context&) override; GetResult get(const Bucket&, const document::FieldSet& fieldSet, const DocumentId& id, Context&) const override; CreateIteratorResult createIterator(const Bucket&, const document::FieldSet& fieldSet, diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp index 2f54cefac29..080d98734bd 100644 --- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp +++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -98,10 +98,10 @@ shafile(const string &baseDir, class StringGenerator { - search::Rand48 &_rnd; + vespalib::Rand48 &_rnd; public: - StringGenerator(search::Rand48 &rnd); + StringGenerator(vespalib::Rand48 &rnd); void rand_string(string &res, uint32_t minLen, uint32_t maxLen); @@ -114,7 +114,7 @@ public: }; -StringGenerator::StringGenerator(search::Rand48 &rnd) +StringGenerator::StringGenerator(vespalib::Rand48 &rnd) : _rnd(rnd) { } @@ -283,7 +283,7 @@ PrefixTextFieldGenerator::generateValue(vespalib::asciistream &doc, uint32_t id) class RandTextFieldGenerator : public FieldGenerator { - search::Rand48 &_rnd; + vespalib::Rand48 &_rnd; uint32_t _numWords; StringArray _strings; uint32_t _minFill; @@ -291,7 +291,7 @@ class RandTextFieldGenerator : public FieldGenerator public: RandTextFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t numWords, uint32_t minFill, uint32_t maxFill); @@ -302,7 +302,7 @@ public: RandTextFieldGenerator::RandTextFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t numWords, uint32_t minFill, uint32_t randFill) @@ -354,7 +354,7 @@ class ModTextFieldGenerator : public FieldGenerator public: ModTextFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const std::vector<uint32_t> &mods); virtual ~ModTextFieldGenerator(); virtual void generateValue(vespalib::asciistream &doc, uint32_t id) override; @@ -362,7 +362,7 @@ public: ModTextFieldGenerator::ModTextFieldGenerator(const string &name, - [[maybe_unused]] search::Rand48 &rnd, + [[maybe_unused]] vespalib::Rand48 &rnd, const std::vector<uint32_t> &mods) : FieldGenerator(name), _mods(mods) @@ -419,13 +419,13 @@ IdTextFieldGenerator::generateValue(vespalib::asciistream &doc, uint32_t id) class RandIntFieldGenerator : public FieldGenerator { - search::Rand48 &_rnd; + vespalib::Rand48 &_rnd; uint32_t _low; uint32_t _count; public: RandIntFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t low, uint32_t count); virtual ~RandIntFieldGenerator(); @@ -436,7 +436,7 @@ public: RandIntFieldGenerator::RandIntFieldGenerator(const string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t low, uint32_t count) : FieldGenerator(name), @@ -617,7 +617,7 @@ class GenTestDocsApp : public SubApp int _optIndex; std::vector<FieldGenerator::SP> _fields; std::vector<uint32_t> _mods; - search::Rand48 _rnd; + vespalib::Rand48 _rnd; string _outFile; bool _headers; bool _json; diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp index f76743b9480..2ae28936476 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp @@ -98,11 +98,32 @@ assertFastSearchAndMoreAttribute(const AttributesConfig::Attribute &attribute, return true; } +bool +assertTensorAttribute(const AttributesConfig::Attribute &attribute, + const vespalib::string &name, const vespalib::string &spec, int max_links_per_node) +{ + if (!EXPECT_EQUAL(attribute.name, name)) { + return false; + } + if (!EXPECT_EQUAL((int)attribute.datatype, (int)AttributesConfig::Attribute::Datatype::TENSOR)) { + return false; + } + if (!EXPECT_EQUAL(attribute.tensortype, spec)) { + return false; + } + if (!EXPECT_TRUE(attribute.index.hnsw.enabled)) { + return false; + } + if (!EXPECT_EQUAL(attribute.index.hnsw.maxlinkspernode, max_links_per_node)) { + return false; + } + return true; +} bool assertAttributes(const AttributesConfig::AttributeVector &attributes) { - if (!EXPECT_EQUAL(4u, attributes.size())) { + if (!EXPECT_EQUAL(6u, attributes.size())) { return false; } if (!assertDefaultAttribute(attributes[0], "a1")) { @@ -117,6 +138,12 @@ assertAttributes(const AttributesConfig::AttributeVector &attributes) if (!assertDefaultAttribute(attributes[3], "a4")) { return false; } + if (!assertTensorAttribute(attributes[4], "tensor1", "tensor(x[100])", 16)) { + return false; + } + if (!assertTensorAttribute(attributes[5], "tensor2", "tensor(x[100])", 16)) { + return false; + } return true; } @@ -124,7 +151,7 @@ assertAttributes(const AttributesConfig::AttributeVector &attributes) bool assertLiveAttributes(const AttributesConfig::AttributeVector &attributes) { - if (!EXPECT_EQUAL(5u, attributes.size())) { + if (!EXPECT_EQUAL(7u, attributes.size())) { return false; } if (!assertFastSearchAttribute(attributes[0], "a0")) { @@ -142,6 +169,12 @@ assertLiveAttributes(const AttributesConfig::AttributeVector &attributes) if (!assertFastSearchAttribute(attributes[4], "a4")) { return false; } + if (!assertTensorAttribute(attributes[5], "tensor1", "tensor(x[100])", 32)) { + return false; + } + if (!assertTensorAttribute(attributes[6], "tensor2", "tensor(x[200])", 32)) { + return false; + } return true; } @@ -149,7 +182,7 @@ assertLiveAttributes(const AttributesConfig::AttributeVector &attributes) bool assertScoutedAttributes(const AttributesConfig::AttributeVector &attributes) { - if (!EXPECT_EQUAL(4u, attributes.size())) { + if (!EXPECT_EQUAL(6u, attributes.size())) { return false; } if (!assertFastSearchAndMoreAttribute(attributes[0], "a1")) { @@ -164,6 +197,12 @@ assertScoutedAttributes(const AttributesConfig::AttributeVector &attributes) if (!assertDefaultAttribute(attributes[3], "a4")) { return false; } + if (!assertTensorAttribute(attributes[4], "tensor1", "tensor(x[100])", 32)) { + return false; + } + if (!assertTensorAttribute(attributes[5], "tensor2", "tensor(x[100])", 16)) { + return false; + } return true; } @@ -199,6 +238,17 @@ setupFastSearchAndMoreAttribute(const vespalib::string name) return attribute; } +AttributesConfig::Attribute +setupTensorAttribute(const vespalib::string &name, const vespalib::string &spec, int max_links_per_node) +{ + AttributesConfig::Attribute attribute; + attribute.name = name; + attribute.datatype = AttributesConfig::Attribute::Datatype::TENSOR; + attribute.tensortype = spec; + attribute.index.hnsw.enabled = true; + attribute.index.hnsw.maxlinkspernode = max_links_per_node; + return attribute; +} void setupDefaultAttributes(AttributesConfigBuilder::AttributeVector &attributes) @@ -207,6 +257,8 @@ setupDefaultAttributes(AttributesConfigBuilder::AttributeVector &attributes) attributes.push_back(setupDefaultAttribute("a2")); attributes.push_back(setupDefaultAttribute("a3")); attributes.push_back(setupDefaultAttribute("a4")); + attributes.push_back(setupTensorAttribute("tensor1", "tensor(x[100])", 16)); + attributes.push_back(setupTensorAttribute("tensor2", "tensor(x[100])", 16)); } @@ -221,6 +273,8 @@ setupLiveAttributes(AttributesConfigBuilder::AttributeVector &attributes) attributes.back().collectiontype = AttributesConfig::Attribute::Collectiontype::ARRAY; attributes.push_back(setupFastSearchAttribute("a4")); attributes.back().createifnonexistent = true; + attributes.push_back(setupTensorAttribute("tensor1", "tensor(x[100])", 32)); + attributes.push_back(setupTensorAttribute("tensor2", "tensor(x[200])", 32)); } } diff --git a/searchcore/src/tests/proton/matching/query_test.cpp b/searchcore/src/tests/proton/matching/query_test.cpp index 96e1eb3cc22..24e1e886351 100644 --- a/searchcore/src/tests/proton/matching/query_test.cpp +++ b/searchcore/src/tests/proton/matching/query_test.cpp @@ -546,8 +546,9 @@ void Test::requireThatNearIteratorsCanBeBuilt() { builder.addStringTerm(string_term, field, 1, Weight(2)); builder.addStringTerm(prefix_term, field, 1, Weight(2)); Node::UP node = builder.build(); - ResolveViewVisitor resolver(ViewResolver(), plain_index_env); - node->accept(resolver); + ViewResolver resolver; + ResolveViewVisitor visitor(resolver, plain_index_env); + node->accept(visitor); ASSERT_TRUE(node.get()); FakeSearchContext context(8); @@ -569,8 +570,9 @@ void Test::requireThatONearIteratorsCanBeBuilt() { builder.addStringTerm(string_term, field, 1, Weight(2)); builder.addStringTerm(prefix_term, field, 1, Weight(2)); Node::UP node = builder.build(); - ResolveViewVisitor resolver(ViewResolver(), plain_index_env); - node->accept(resolver); + ViewResolver resolver; + ResolveViewVisitor visitor(resolver, plain_index_env); + node->accept(visitor); ASSERT_TRUE(node.get()); FakeSearchContext context(8); @@ -593,8 +595,9 @@ void Test::requireThatPhraseIteratorsCanBeBuilt() { builder.addStringTerm(prefix_term, field, 1, Weight(2)); builder.addStringTerm(suffix_term, field, 1, Weight(2)); Node::UP node = builder.build(); - ResolveViewVisitor resolver(ViewResolver(), plain_index_env); - node->accept(resolver); + ViewResolver resolver; + ResolveViewVisitor visitor(resolver, plain_index_env); + node->accept(visitor); ASSERT_TRUE(node.get()); FakeSearchContext context(9); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt index f7a6ffe7189..550d0a1c4c4 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/attribute/CMakeLists.txt @@ -13,6 +13,7 @@ vespa_add_library(searchcore_attribute STATIC attribute_manager_initializer.cpp attribute_populator.cpp attribute_spec.cpp + attribute_type_matcher.cpp attribute_usage_filter.cpp attribute_usage_sampler_context.cpp attribute_usage_sampler_functor.cpp diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.cpp new file mode 100644 index 00000000000..c99a6ae9e6e --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.cpp @@ -0,0 +1,34 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "attribute_type_matcher.h" +#include <vespa/searchcommon/attribute/config.h> + +using search::attribute::BasicType; + +namespace proton +{ + +bool +AttributeTypeMatcher::operator()(const search::attribute::Config &oldConfig, const search::attribute::Config &newConfig) const +{ + if ((oldConfig.basicType() != newConfig.basicType()) || + (oldConfig.collectionType() != newConfig.collectionType())) { + return false; + } + if (newConfig.basicType().type() == BasicType::Type::TENSOR) { + if (oldConfig.tensorType() != newConfig.tensorType()) { + return false; + } + } + if (newConfig.basicType().type() == BasicType::Type::PREDICATE) { + using Params = search::attribute::PersistentPredicateParams; + const Params &oldParams = oldConfig.predicateParams(); + const Params &newParams = newConfig.predicateParams(); + if (!(oldParams == newParams)) { + return false; + } + } + return true; +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.h new file mode 100644 index 00000000000..bbe9479083b --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_type_matcher.h @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace search::attribute { class Config; } + +namespace proton +{ + +/** + * Class to check if attribute types are compatible or not. + */ +class AttributeTypeMatcher +{ +public: + AttributeTypeMatcher() = default; + ~AttributeTypeMatcher() = default; + bool operator()(const search::attribute::Config &oldConfig, + const search::attribute::Config &newConfig) const; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index 5ff40fa5360..214db2c2d13 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -4,6 +4,7 @@ #include "attribute_directory.h" #include "attributedisklayout.h" #include "attributemanager.h" +#include "attribute_type_matcher.h" #include "imported_attributes_context.h" #include "imported_attributes_repo.h" #include "sequential_attributes_initializer.h" @@ -41,25 +42,8 @@ namespace { bool matchingTypes(const AttributeVector::SP &av, const search::attribute::Config &newConfig) { if (av) { - const auto &oldConfig = av->getConfig(); - if ((oldConfig.basicType() != newConfig.basicType()) || - (oldConfig.collectionType() != newConfig.collectionType())) { - return false; - } - if (newConfig.basicType().type() == BasicType::TENSOR) { - if (oldConfig.tensorType() != newConfig.tensorType()) { - return false; - } - } - if (newConfig.basicType().type() == BasicType::PREDICATE) { - using Params = search::attribute::PersistentPredicateParams; - const Params &oldParams = oldConfig.predicateParams(); - const Params &newParams = newConfig.predicateParams(); - if (!(oldParams == newParams)) { - return false; - } - } - return true; + AttributeTypeMatcher matching_types; + return matching_types(av->getConfig(), newConfig); } else { return false; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp index 2f6cf9e6208..986571d07e1 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributesconfigscout.cpp @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attributesconfigscout.h" +#include "attribute_type_matcher.h" #include <vespa/searchlib/attribute/configconverter.h> using search::attribute::ConfigConverter; @@ -29,6 +30,7 @@ AttributesConfigScout::adjust(AttributesConfig::Attribute &attr, attr.huge = liveAttr.huge; // Note: Predicate attributes only handle changes for the dense-posting-list-threshold config. attr.densepostinglistthreshold = liveAttr.densepostinglistthreshold; + attr.index = liveAttr.index; } @@ -41,8 +43,8 @@ AttributesConfigScout::adjust(AttributesConfig::Attribute &attr) const auto &liveAttr = _live.attribute[it->second]; search::attribute::Config liveCfg = ConfigConverter::convert(liveAttr); - if (cfg.basicType() == liveCfg.basicType() && - cfg.collectionType() == liveCfg.collectionType()) { + AttributeTypeMatcher matching_types; + if (matching_types(cfg, liveCfg)) { adjust(attr, liveAttr); } } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 3e9ef787f74..c36ee8e474a 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -17,7 +17,7 @@ #include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/vespalib/btree/btreenodestore.hpp> #include <vespa/vespalib/btree/btreeroot.hpp> -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/rcuvector.hpp> #include "document_meta_store_versions.h" diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp index 2a3fd14c6d2..7d85c01ee74 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentmetastoresaver.h" -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include "document_meta_store_versions.h" #include <vespa/searchlib/attribute/iattributesavetarget.h> diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index d058104bf1b..6bce791914c 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1416,7 +1416,8 @@ "public com.yahoo.searchlib.rankingexpression.ExpressionFunction getFunction(java.lang.String)", "protected com.google.common.collect.ImmutableMap functions()", "public java.lang.String getBinding(java.lang.String)", - "public com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)" + "public com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)", + "public com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withoutBindings()" ], "fields": [] }, @@ -1573,7 +1574,9 @@ "public void addArgumentTypeSerialization(java.lang.String, java.lang.String, com.yahoo.tensor.TensorType)", "public void addFunctionTypeSerialization(java.lang.String, com.yahoo.tensor.TensorType)", "public com.yahoo.searchlib.rankingexpression.rule.SerializationContext withBindings(java.util.Map)", + "public com.yahoo.searchlib.rankingexpression.rule.SerializationContext withoutBindings()", "public java.util.Map serializedFunctions()", + "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withoutBindings()", "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)" ], "fields": [] diff --git a/searchlib/src/apps/tests/btreestress_test.cpp b/searchlib/src/apps/tests/btreestress_test.cpp index e6ef0740fc3..a39416c2227 100644 --- a/searchlib/src/apps/tests/btreestress_test.cpp +++ b/searchlib/src/apps/tests/btreestress_test.cpp @@ -6,7 +6,7 @@ #include <vespa/vespalib/btree/btreenodeallocator.h> #include <vespa/vespalib/btree/btree.h> #include <vespa/vespalib/btree/btreestore.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/vespalib/btree/btreenode.hpp> @@ -37,7 +37,7 @@ struct Fixture MyTreeIterator _writeItr; vespalib::ThreadStackExecutor _writer; // 1 write thread vespalib::ThreadStackExecutor _readers; // multiple reader threads - search::Rand48 _rnd; + vespalib::Rand48 _rnd; uint32_t _keyLimit; std::atomic<long> _readSeed; std::atomic<long> _doneWriteWork; @@ -135,7 +135,7 @@ Fixture::remove(uint32_t key) void Fixture::readWork(uint32_t cnt) { - search::Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(++_readSeed); uint32_t i; for (i = 0; i < cnt && _stopRead.load() == 0; ++i) { @@ -159,7 +159,7 @@ Fixture::readWork() void Fixture::writeWork(uint32_t cnt) { - search::Rand48 &rnd(_rnd); + vespalib::Rand48 &rnd(_rnd); for (uint32_t i = 0; i < cnt; ++i) { uint32_t key = rnd.lrand48() % _keyLimit; if ((rnd.lrand48() & 1) == 0) { diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 6bbd93bda84..041335b7ed1 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -20,7 +20,7 @@ #include <vespa/searchlib/queryeval/fake_searchable.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/test/index/mock_field_length_inspector.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> @@ -201,7 +201,7 @@ struct Fixture { uint32_t _readThreads; vespalib::ThreadStackExecutor _writer; // 1 write thread vespalib::ThreadStackExecutor _readers; // multiple reader threads - search::Rand48 _rnd; + vespalib::Rand48 _rnd; uint32_t _keyLimit; std::atomic<long> _readSeed; std::atomic<long> _doneWriteWork; @@ -285,7 +285,7 @@ Fixture::~Fixture() void Fixture::readWork(uint32_t cnt) { - search::Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(++_readSeed); uint32_t i; uint32_t emptyCount = 0; @@ -345,7 +345,7 @@ Fixture::readWork() void Fixture::writeWork(uint32_t cnt) { - search::Rand48 &rnd(_rnd); + vespalib::Rand48 &rnd(_rnd); for (uint32_t i = 0; i < cnt; ++i) { uint32_t key = rnd.lrand48() % _keyLimit; if ((rnd.lrand48() & 1) == 0) { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 674571ff73e..9f900ffed36 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -3,7 +3,6 @@ package com.yahoo.searchlib.rankingexpression; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.yahoo.log.event.Collection; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; @@ -134,7 +133,10 @@ public class ExpressionFunction { for (int i = 0; i < arguments.size() && i < argumentValues.size(); ++i) { argumentBindings.put(arguments.get(i), argumentValues.get(i).toString(new StringBuilder(), context, path, null).toString()); } - return new Instance(toSymbol(argumentBindings), body.getRoot().toString(new StringBuilder(), context.withBindings(argumentBindings), path, null).toString()); + context = argumentBindings.isEmpty() ? context.withoutBindings() : context.withBindings(argumentBindings); + String symbol = toSymbol(argumentBindings); + String expressionString = body.getRoot().toString(new StringBuilder(), context, path, null).toString(); + return new Instance(symbol, expressionString); } /** diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java index 83aabada8f0..6717bec0258 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java @@ -68,4 +68,9 @@ public class FunctionReferenceContext { return new FunctionReferenceContext(this.functions, bindings); } + /** Returns a fresh context without bindings */ + public FunctionReferenceContext withoutBindings() { + return new FunctionReferenceContext(this.functions); + } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java index 8fec3603f3e..335d3861d9d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ReferenceNode.java @@ -5,7 +5,6 @@ import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.Context; -import com.yahoo.searchlib.rankingexpression.evaluation.StringValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; @@ -13,7 +12,7 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.ArrayDeque; import java.util.Deque; import java.util.List; -import java.util.Optional; +import java.util.Map; /** * A node referring either to a value in the context or to a named ranking expression function. @@ -84,13 +83,25 @@ public final class ReferenceNode extends CompositeNode { if (path.contains(myPath)) throw new IllegalStateException("Cycle in ranking expression function: " + path); path.addLast(myPath); - ExpressionFunction.Instance instance = function.expand(context, getArguments().expressions(), path); + + String functionName = getName(); + String functionPropertyName = RankingExpression.propertyName(functionName); + boolean alreadySerialized = context.serializedFunctions().containsKey(functionPropertyName); + + if ( ! alreadySerialized || getArguments().size() > 0) { + ExpressionFunction.Instance instance = function.expand(context, getArguments().expressions(), path); + functionName = instance.getName(); + + context.addFunctionSerialization(RankingExpression.propertyName(functionName), instance.getExpressionString()); + for (Map.Entry<String, TensorType> argumentType : function.argumentTypes().entrySet()) + context.addArgumentTypeSerialization(functionName, argumentType.getKey(), argumentType.getValue()); + if (function.returnType().isPresent()) + context.addFunctionTypeSerialization(functionName, function.returnType().get()); + } path.removeLast(); - context.addFunctionSerialization(RankingExpression.propertyName(instance.getName()), instance.getExpressionString()); - return string.append("rankingExpression(").append(instance.getName()).append(')'); + return string.append("rankingExpression(").append(functionName).append(')'); } - // Not resolved in this context: output as-is return reference.toString(string, context, path, parent); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index d7807caa2b6..92889d6607b 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -95,6 +95,12 @@ public class SerializationContext extends FunctionReferenceContext { return new SerializationContext(functions(), bindings, this.serializedFunctions); } + /** Returns a fresh context without bindings */ + @Override + public SerializationContext withoutBindings() { + return new SerializationContext(functions(), null, this.serializedFunctions); + } + public Map<String, String> serializedFunctions() { return serializedFunctions; } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java index 6e1cdf52158..befe2179dc1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java @@ -333,10 +333,16 @@ public class TensorFunctionNode extends CompositeNode { /** Returns a new context with the bindings replaced by the given bindings */ @Override public ExpressionToStringContext withBindings(Map<String, String> bindings) { - return new ExpressionToStringContext(new SerializationContext(wrappedSerializationContext.functions().values(), bindings), - wrappedToStringContext, path, parent); + SerializationContext serializationContext = new SerializationContext(functions(), bindings, serializedFunctions()); + return new ExpressionToStringContext(serializationContext, wrappedToStringContext, path, parent); } + /** Returns a fresh context without bindings */ + @Override + public SerializationContext withoutBindings() { + SerializationContext serializationContext = new SerializationContext(functions(), null, serializedFunctions()); + return new ExpressionToStringContext(serializationContext, null, path, parent); + } } /** Turns an EvaluationContext into a Context */ diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java index a541eac2421..fed4f60e983 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/transform/ConstantDereferencer.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.transform; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; @@ -28,8 +28,15 @@ public class ConstantDereferencer extends ExpressionTransformer<TransformContext return node; } + /** Returns true if the given reference is an attribute, constant or query feature */ + private static boolean isSimpleFeature(Reference reference) { + if ( ! reference.isSimple()) return false; + String name = reference.name(); + return name.equals("attribute") || name.equals("constant") || name.equals("query"); + } + private ExpressionNode transformFeature(ReferenceNode node, TransformContext context) { - if (!node.getArguments().isEmpty()) + if ( ! node.getArguments().isEmpty() && ! isSimpleFeature(node.reference())) return transformArguments(node, context); else return transformConstantReference(node, context); @@ -44,7 +51,14 @@ public class ConstantDereferencer extends ExpressionTransformer<TransformContext } private ExpressionNode transformConstantReference(ReferenceNode node, TransformContext context) { - Value value = context.constants().get(node.getName()); + String name = node.getName(); + if (node.reference().name().equals("constant")) { + ExpressionNode arg = node.getArguments().expressions().get(0); + if (arg instanceof ReferenceNode) { + name = ((ReferenceNode)arg).getName(); + } + } + Value value = context.constants().get(name); // works if "constant(...)" is added if (value == null || value.type().rank() > 0) { return node; // not a number constant reference } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java index 807eb3aa7ce..6a9108311b1 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationTestCase.java @@ -10,7 +10,6 @@ import com.yahoo.searchlib.rankingexpression.rule.ArithmeticOperator; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.IfNode; -import com.yahoo.tensor.TensorType; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -390,6 +389,130 @@ public class EvaluationTestCase { } @Test + public void testTile() { + EvaluationTester tester = new EvaluationTester(); + + tester.assertEvaluates("tensor(d0[2],d1[4]):[1,2,1,2,3,4,3,4]", + "tensor(d0[2],d1[4])(tensor0{input0:(d0 % 2), input1:(d1 % 2) } )", + "tensor(input0[2],input1[2]):[1, 2, 3, 4]", + "tensor(repeats0[2]):[1,2]"); + + tester.assertEvaluates("tensor(d0[6],d1[2]):[1,2,3,4,1,2,3,4,1,2,3,4]", + "tensor(d0[6],d1[2])(tensor0{input0:(d0 % 2), input1:(d1 % 2) } )", + "tensor(input0[2],input1[2]):[1, 2, 3, 4]", + "tensor(repeats0[2]):[3,1]"); + } + + @Test + public void testReshape() { + EvaluationTester tester = new EvaluationTester(); + + tester.assertEvaluates("tensor(d0[4]):[1,2,3,4]", + "tensor(d0[4])(tensor0{a0:(d0 / 2), a1:(d0 % 2)})", + "tensor(a0[2],a1[2]):[1,2,3,4]", + "tensor(d0[1]):[4]"); + + tester.assertEvaluates("tensor(d0[2],d1[2]):[1,2,3,4]", + "tensor(d0[2],d1[2])(tensor0{a0:(d0), a1:(d1)})", + "tensor(a0[2],a1[2]):[1,2,3,4]", + "tensor(d0[2]):[2,2]"); + + tester.assertEvaluates("tensor(d0[2],d1[1],d2[2]):[1,2,3,4]", + "tensor(d0[2],d1[1],d2[2])(tensor0{a0:(d0), a1:(d2)})", + "tensor(a0[2],a1[2]):[1,2,3,4]", + "tensor(d0[3]):[2,1,2]"); + + tester.assertEvaluates("tensor(d0[3],d1[2]):[1,2,3,4,5,6]", + "tensor(d0[3],d1[2])(tensor0{a0:0, a1:((d0 * 2 + d1) / 3), a2:((d0 * 2 + d1) % 3) })", + "tensor(a0[1],a1[2],a2[3]):[1,2,3,4,5,6]", + "tensor(d0[2]):[3,2]"); + + tester.assertEvaluates("tensor(d0[3],d1[2],d2[1],d3[1]):[1,2,3,4,5,6]", + "tensor(d0[3],d1[2],d2[1],d3[1])(tensor0{a0:0, a1:((d0 * 2 + d1) / 3), a2:((d0 * 2 + d1) % 3) })", + "tensor(a0[1],a1[2],a2[3]):[1,2,3,4,5,6]", + "tensor(d0[4]):[3,2,-1,1]"); + + } + + @Test + public void testMatmul() { + EvaluationTester tester = new EvaluationTester(); + + tester.assertEvaluates("tensor():{91}", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d0)", + "tensor(d0[6]):[1,2,3,4,5,6]", + "tensor(d0[6]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d1[2]):[22, 28]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d0)", + "tensor(d0[3]):[1,2,3]", + "tensor(d0[3],d1[2]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d1[2]):[22, 28]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d0)", + "tensor(d0[3],d1[2]):[1,2,3,4,5,6]", + "tensor(d0[3]):[1,2,3]"); + + tester.assertEvaluates("tensor(d0[2],d2[2]):[22,28,49,64]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d1)", + "tensor(d0[2],d1[3]):[1,2,3,4,5,6]", + "tensor(d1[3],d2[2]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d0[1],d1[2],d3[2]):[22,28,49,64]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d2)", + "tensor(d0[1],d1[2],d2[3]):[1,2,3,4,5,6]", + "tensor(d2[3],d3[2]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d0[1],d1[2],d3[2]):[22,28,49,64]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d2)", + "tensor(d1[2],d2[3]):[1,2,3,4,5,6]", + "tensor(d0[1],d2[3],d3[2]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d0[2],d1[2],d3[2]):[22,28,49,64,58,64,139,154]", + "reduce(join(tensor0{d0:0}, tensor1, f(x,y)(x*y)), sum, d2)", // notice peek + "tensor(d0[1],d1[2],d2[3]):[1,2,3,4,5,6]", + "tensor(d0[2],d2[3],d3[2]):[1,2,3,4,5,6,7,8,9,10,11,12]"); + + tester.assertEvaluates("tensor(d0[2],d1[2],d2[2],d4[2]):[22,28,49,64,58,64,139,154,76,100,103,136,220,244,301,334]", + "reduce(join(tensor0{d1:0}, tensor1{d0:0}, f(x,y)(x*y)), sum, d3)", // notice peeks + "tensor(d0[2],d1[1],d2[2],d3[3]):[1,2,3,4,5,6,7,8,9,10,11,12]", + "tensor(d0[1],d1[2],d3[3],d4[2]):[1,2,3,4,5,6,7,8,9,10,11,12]"); + + tester.assertEvaluates("tensor(d0[1],d1[4],d2[2],d4[2]):[22,28,49,64,220,244,301,334,634,676,769,820,1264,1324,1453,1522]", + "reduce(join(tensor0, tensor1, f(x,y)(x*y)), sum, d3)", + "tensor(d0[1],d1[4],d2[2],d3[3]):[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24]", + "tensor(d0[1],d1[4],d3[3],d4[2]):[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24]"); + } + + @Test + public void testSplit() { + EvaluationTester tester = new EvaluationTester(); + + tester.assertEvaluates("tensor(d0[3]):[1,2,3]", + "tensor(d0[3])(tensor0{input0:(d0)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + tester.assertEvaluates("tensor(d0[3]):[4,5,6]", + "tensor(d0[3])(tensor0{input0:(d0+3)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + tester.assertEvaluates("tensor(d0[4]):[3,4,5,6]", + "tensor(d0[4])(tensor0{input0:(d0+2)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + tester.assertEvaluates("tensor(d0[2]):[3,4]", + "tensor(d0[2])(tensor0{input0:(d0+2)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + tester.assertEvaluates("tensor(d0[2]):[5,6]", + "tensor(d0[2])(tensor0{input0:(d0+4)} )", + "tensor(input0[6]):[1,2,3,4,5,6]"); + + tester.assertEvaluates("tensor(d0[1],d1[3]):[1,2,3]", + "tensor(d0[1],d1[3])(tensor0{input0:(d0), input1:(d1)} )", + "tensor(input0[2],input1[3]):[[1,2,3],[4,5,6]]"); + tester.assertEvaluates("tensor(d0[1],d1[3]):[4,5,6]", + "tensor(d0[1],d1[3])(tensor0{input0:(d0+1), input1:(d1)} )", + "tensor(input0[2],input1[3]):[[1,2,3],[4,5,6]]"); + } + + @Test public void testTake() { EvaluationTester tester = new EvaluationTester(); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java index f4b1b0ceee2..cf761aac2d3 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/transform/SimplifierTestCase.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.rankingexpression.transform; -import com.yahoo.log.event.Collection; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.evaluation.Context; import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; diff --git a/searchlib/src/tests/aggregator/perdocexpr.cpp b/searchlib/src/tests/aggregator/perdocexpr.cpp index 41465f991e3..039b3ae1fc0 100644 --- a/searchlib/src/tests/aggregator/perdocexpr.cpp +++ b/searchlib/src/tests/aggregator/perdocexpr.cpp @@ -762,7 +762,9 @@ TEST("testDivExpressions") { ToRawFunctionNode e(MU<ConstantNode>(MU<Int64ResultNode>(238686))); e.prepare(false); e.execute(); - EXPECT_EQUAL(strcmp(static_cast<const RawResultNode &>(e.getResult()).get().c_str(), "238686"), 0); + auto raw_result = static_cast<const RawResultNode &>(e.getResult()).get(); + EXPECT_EQUAL(6u, raw_result.size()); + EXPECT_EQUAL(strncmp(raw_result.c_str(), "238686", 6u), 0); } { diff --git a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp index 8d13987bb4f..eeabbf23c27 100644 --- a/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp +++ b/searchlib/src/tests/attribute/attributefilewriter/attributefilewriter_test.cpp @@ -6,7 +6,7 @@ #include <vespa/searchlib/attribute/attributefilebufferwriter.h> #include <vespa/searchlib/attribute/attribute_header.h> #include <vespa/searchlib/util/fileutil.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> @@ -77,7 +77,7 @@ TEST_F("Test that buffer writer passes on written data", Fixture) const size_t writerBufferSize = AttributeFileBufferWriter::BUFFER_SIZE; EXPECT_GREATER(mysize * sizeof(int), writerBufferSize); a.reserve(mysize); - search::Rand48 rnd; + vespalib::Rand48 rnd; for (uint32_t i = 0; i < mysize; ++i) { a.emplace_back(rnd.lrand48()); } diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index 41313fc7c53..d8761f69d71 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -19,7 +19,7 @@ #include <vespa/searchlib/parsequery/parse.h> #include <vespa/searchlib/util/randomgenerator.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/compress.h> #include <vespa/searchlib/attribute/attributevector.hpp> @@ -227,7 +227,7 @@ void EnumeratedSaveTest::populate(IntegerAttribute &v, unsigned seed, BasicType bt) { - search::Rand48 rnd; + vespalib::Rand48 rnd; IntegerAttribute::largeint_t mask(std::numeric_limits <IntegerAttribute::largeint_t>::max()); switch (bt.type()) { @@ -284,7 +284,7 @@ EnumeratedSaveTest::populate(FloatingPointAttribute &v, unsigned seed, BasicType bt) { (void) bt; - search::Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(seed); int weight = 1; for(size_t i(0), m(v.getNumDocs()); i < m; i++) { diff --git a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp index 2f74e9d31f6..a936ba124ee 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp @@ -3,7 +3,7 @@ #include <vespa/searchlib/attribute/multi_value_mapping.h> #include <vespa/searchlib/attribute/multi_value_mapping.hpp> #include <vespa/searchlib/attribute/not_implemented_attribute.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/test/insertion_operators.h> @@ -151,7 +151,7 @@ using IntMappingTest = MappingTestBase<int>; class CompactionIntMappingTest : public MappingTestBase<int> { - search::Rand48 _rnd; + vespalib::Rand48 _rnd; std::map<uint32_t, std::vector<int>> _refMapping; public: CompactionIntMappingTest() diff --git a/searchlib/src/tests/attribute/postinglist/postinglist.cpp b/searchlib/src/tests/attribute/postinglist/postinglist.cpp index 78b93e2b78b..37d76e4a9e8 100644 --- a/searchlib/src/tests/attribute/postinglist/postinglist.cpp +++ b/searchlib/src/tests/attribute/postinglist/postinglist.cpp @@ -7,7 +7,7 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/datastore/datastore.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/testkit/testapp.h> #include <set> #include <map> @@ -143,7 +143,7 @@ private: PostingList *_intPostings; STLValueTree *_stlTree; - Rand48 _randomGenerator; + vespalib::Rand48 _randomGenerator; uint32_t _generation; void diff --git a/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp b/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp index c746a0aa120..103551dd93a 100644 --- a/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp +++ b/searchlib/src/tests/attribute/save_target/attribute_save_target_test.cpp @@ -7,7 +7,7 @@ #include <vespa/searchlib/test/directory_handler.h> #include <vespa/searchlib/util/fileutil.h> #include <vespa/vespalib/gtest/gtest.h> -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 39a7e53ca8c..0dc8db870d9 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -22,7 +22,7 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/log/log.h> LOG_SETUP("tensorattribute_test"); @@ -215,6 +215,7 @@ struct Fixture { Config _cfg; vespalib::string _name; vespalib::string _typeSpec; + bool _use_mock_index; std::unique_ptr<NearestNeighborIndexFactory> _index_factory; std::shared_ptr<TensorAttribute> _tensorAttr; std::shared_ptr<AttributeVector> _attr; @@ -229,27 +230,45 @@ struct Fixture { _cfg(BasicType::TENSOR, CollectionType::SINGLE), _name(attr_name), _typeSpec(typeSpec), - _index_factory(std::make_unique<DefaultNearestNeighborIndexFactory>()), + _use_mock_index(use_mock_index), + _index_factory(), _tensorAttr(), _attr(), _denseTensors(false), _useDenseTensorAttribute(useDenseTensorAttribute) { - _cfg.setTensorType(ValueType::from_spec(typeSpec)); + if (enable_hnsw_index) { + _cfg.set_hnsw_index_params(HnswIndexParams(4, 20, DistanceMetric::Euclidean)); + } + setup(); + } + + ~Fixture() {} + + void setup() { + _cfg.setTensorType(ValueType::from_spec(_typeSpec)); if (_cfg.tensorType().is_dense()) { _denseTensors = true; } - if (enable_hnsw_index) { - _cfg.set_hnsw_index_params(HnswIndexParams(4, 20, DistanceMetric::Euclidean)); - if (use_mock_index) { - _index_factory = std::make_unique<MockNearestNeighborIndexFactory>(); - } + if (_use_mock_index) { + _index_factory = std::make_unique<MockNearestNeighborIndexFactory>(); + } else { + _index_factory = std::make_unique<DefaultNearestNeighborIndexFactory>(); } _tensorAttr = makeAttr(); _attr = _tensorAttr; _attr->addReservedDoc(); } - ~Fixture() {} + + void set_hnsw_index_params(const HnswIndexParams ¶ms) { + _cfg.set_hnsw_index_params(params); + setup(); + } + + void disable_hnsw_index() { + _cfg.clear_hnsw_index_params(); + setup(); + } std::shared_ptr<TensorAttribute> makeAttr() { if (_useDenseTensorAttribute) { @@ -364,6 +383,9 @@ struct Fixture { return denseSpec; } + void set_example_tensors(); + void assert_example_tensors(); + void save_example_tensors_with_mock_index(); void testEmptyAttribute(); void testSetTensorValue(); void testSaveLoad(); @@ -372,6 +394,30 @@ struct Fixture { void testEmptyTensor(); }; + +void +Fixture::set_example_tensors() +{ + set_tensor(1, vec_2d(3, 5)); + set_tensor(2, vec_2d(7, 9)); +} + +void +Fixture::assert_example_tensors() +{ + assertGetTensor(vec_2d(3, 5), 1); + assertGetTensor(vec_2d(7, 9), 2); +} + +void +Fixture::save_example_tensors_with_mock_index() +{ + set_example_tensors(); + mock_index().save_index_with_value(123); + save(); + EXPECT_TRUE(vespalib::fileExists(_name + ".nnidx")); +} + void Fixture::testEmptyAttribute() { @@ -664,11 +710,7 @@ TEST_F("Memory usage is extracted from index when updating stats on attribute", TEST_F("Nearest neighbor index can be saved to disk and then loaded from file", DenseTensorAttributeMockIndex) { - f.set_tensor(1, vec_2d(3, 5)); - f.set_tensor(2, vec_2d(7, 9)); - f.mock_index().save_index_with_value(123); - f.save(); - EXPECT_TRUE(vespalib::fileExists(attr_name + ".nnidx")); + f.save_example_tensors_with_mock_index(); f.load(); // index is loaded from saved file auto& index = f.mock_index(); @@ -678,8 +720,7 @@ TEST_F("Nearest neighbor index can be saved to disk and then loaded from file", TEST_F("onLoad() reconstructs nearest neighbor index if save file does not exists", DenseTensorAttributeMockIndex) { - f.set_tensor(1, vec_2d(3, 5)); - f.set_tensor(2, vec_2d(7, 9)); + f.set_example_tensors(); f.save(); EXPECT_FALSE(vespalib::fileExists(attr_name + ".nnidx")); @@ -689,5 +730,36 @@ TEST_F("onLoad() reconstructs nearest neighbor index if save file does not exist index.expect_adds({{1, {3, 5}}, {2, {7, 9}}}); } +TEST_F("onLoads() ignores saved nearest neighbor index if not enabled in config", DenseTensorAttributeMockIndex) +{ + f.save_example_tensors_with_mock_index(); + f.disable_hnsw_index(); + f.load(); + f.assert_example_tensors(); + EXPECT_EQUAL(f.as_dense_tensor().nearest_neighbor_index(), nullptr); +} + +TEST_F("onLoad() ignores saved nearest neighbor index if major index parameters are changed", DenseTensorAttributeMockIndex) +{ + f.save_example_tensors_with_mock_index(); + f.set_hnsw_index_params(HnswIndexParams(5, 20, DistanceMetric::Euclidean)); + f.load(); + f.assert_example_tensors(); + auto& index = f.mock_index(); + EXPECT_EQUAL(0, index.get_index_value()); + index.expect_adds({{1, {3, 5}}, {2, {7, 9}}}); +} + +TEST_F("onLoad() uses saved nearest neighbor index if only minor index parameters are changed", DenseTensorAttributeMockIndex) +{ + f.save_example_tensors_with_mock_index(); + f.set_hnsw_index_params(HnswIndexParams(4, 21, DistanceMetric::Euclidean)); + f.load(); + f.assert_example_tensors(); + auto& index = f.mock_index(); + EXPECT_EQUAL(123, index.get_index_value()); + index.expect_adds({}); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index 409cc9f2725..db29ab9881f 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -8,7 +8,7 @@ #include <vespa/searchlib/common/bitvectoriterator.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <algorithm> using namespace search; @@ -81,7 +81,7 @@ myCountInterval(const BitVector &bv, uint32_t low, uint32_t high) } void -scan(uint32_t count, uint32_t offset, uint32_t size, Rand48 &rnd) +scan(uint32_t count, uint32_t offset, uint32_t size, vespalib::Rand48 &rnd) { std::vector<uint32_t> lids; lids.reserve(count); @@ -110,7 +110,7 @@ scan(uint32_t count, uint32_t offset, uint32_t size, Rand48 &rnd) void scanWithOffset(uint32_t offset) { - Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(32); scan(10, offset, 1000000, rnd); diff --git a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp index ef55ea60c2b..15a53497db9 100644 --- a/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp +++ b/searchlib/src/tests/diskindex/fieldwriter/fieldwriter_test.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/common/bitvector.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/searchlib/test/fakedata/fakeword.h> #include <vespa/searchlib/test/fakedata/fakewordset.h> #include <vespa/searchlib/index/docidandfeatures.h> @@ -99,7 +99,7 @@ private: FakeWordSet _wordSet; FakeWordSet _wordSet2; public: - search::Rand48 _rnd; + vespalib::Rand48 _rnd; private: void Usage(); diff --git a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp index e914fe7c559..6cc2f318f4b 100644 --- a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp +++ b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/bitcompression/compression.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/bitcompression/countcompression.h> #include <vespa/searchlib/bitcompression/pagedict4.h> @@ -54,7 +54,7 @@ using RandReader = search::diskindex::test::PageDict4MemRandReader; class PageDict4TestApp : public FastOS_Application { public: - search::Rand48 _rnd; + vespalib::Rand48 _rnd; bool _stress; bool _emptyWord; bool _firstWordForcedCommon; @@ -206,7 +206,7 @@ deDup(std::vector<uint32_t> &v) static WordIndexCounts -makeIndex(search::Rand48 &rnd, bool forceCommon) +makeIndex(vespalib::Rand48 &rnd, bool forceCommon) { uint64_t bitLength = 10; uint32_t numDocs = 1; @@ -219,7 +219,7 @@ makeIndex(search::Rand48 &rnd, bool forceCommon) void -makeIndexes(search::Rand48 &rnd, +makeIndexes(vespalib::Rand48 &rnd, WordIndexCounts &counts, bool forceCommon) { @@ -229,7 +229,7 @@ makeIndexes(search::Rand48 &rnd, static void makeWords(std::vector<WordCounts> &v, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint32_t numWordIds, uint32_t tupleCount, bool emptyWord, @@ -355,7 +355,7 @@ checkCounts(const std::string &word, void testWords(const std::string &logname, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, uint64_t numWordIds, uint32_t tupleCount, uint32_t chunkSize, diff --git a/searchlib/src/tests/features/beta/beta_features.cpp b/searchlib/src/tests/features/beta/beta_features.cpp index 617bf2fb073..89a72fab6fc 100644 --- a/searchlib/src/tests/features/beta/beta_features.cpp +++ b/searchlib/src/tests/features/beta/beta_features.cpp @@ -31,7 +31,7 @@ #include <vespa/searchlib/fef/featurenamebuilder.h> #include <vespa/searchlib/fef/indexproperties.h> #include <vespa/searchlib/fef/test/plugin/setup.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/searchlib/fef/test/ftlib.h> #include <vespa/vespalib/util/stringfmt.h> diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 1c07c81bc2f..3f71c9f85ea 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -40,7 +40,7 @@ #include <vespa/searchlib/fef/queryproperties.h> #include <vespa/searchlib/fef/test/plugin/setup.h> #include <vespa/searchlib/fef/test/dummy_dependency_handler.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/util/string_hash.h> @@ -1747,7 +1747,7 @@ Test::testRandom() FtFeatureTest ft(_factory, "random"); ft.getIndexEnv().getProperties().add("random.seed", "100"); ASSERT_TRUE(ft.setup()); - search::Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(100); for (uint32_t i = 0; i < 5; ++i) { feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u); @@ -1770,7 +1770,7 @@ Test::testRandom() FtFeatureTest ft(_factory, "random.match"); ft.getQueryEnv().getProperties().add("random.match.seed", "100"); ASSERT_TRUE(ft.setup()); - search::Rand48 rnd; + vespalib::Rand48 rnd; for (uint32_t i = 1; i <= 5; ++i) { rnd.srand48(100 + i); // seed + lid feature_t exp = static_cast<feature_t>(rnd.lrand48()) / static_cast<feature_t>(0x80000000u); diff --git a/searchlib/src/tests/postinglistbm/posting_list_test.cpp b/searchlib/src/tests/postinglistbm/posting_list_test.cpp index a7d810cb1a1..845201772f8 100644 --- a/searchlib/src/tests/postinglistbm/posting_list_test.cpp +++ b/searchlib/src/tests/postinglistbm/posting_list_test.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/test/fakedata/fakeword.h> #include <vespa/searchlib/test/fakedata/fakewordset.h> #include <vespa/searchlib/test/fakedata/fpfactory.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/gtest/gtest.h> #include <cinttypes> @@ -66,7 +66,7 @@ struct PostingListTest : public ::testing::Test { FakeWordUP word3; FakeWordUP word4; FakeWordUP word5; - search::Rand48 rnd; + vespalib::Rand48 rnd; PostingListTest() : num_docs(36000), diff --git a/searchlib/src/tests/postinglistbm/postinglistbm.cpp b/searchlib/src/tests/postinglistbm/postinglistbm.cpp index 16b8e9cd7f5..0ac3f15e015 100644 --- a/searchlib/src/tests/postinglistbm/postinglistbm.cpp +++ b/searchlib/src/tests/postinglistbm/postinglistbm.cpp @@ -10,7 +10,7 @@ #include <vespa/searchlib/test/fakedata/fakeword.h> #include <vespa/searchlib/test/fakedata/fakewordset.h> #include <vespa/searchlib/test/fakedata/fpfactory.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/log/log.h> @@ -42,7 +42,7 @@ private: bool _unpack; public: - search::Rand48 _rnd; + vespalib::Rand48 _rnd; public: PostingListBM(); diff --git a/searchlib/src/tests/postinglistbm/stress_runner.cpp b/searchlib/src/tests/postinglistbm/stress_runner.cpp index 9c78515a03f..fd12e9b5aa9 100644 --- a/searchlib/src/tests/postinglistbm/stress_runner.cpp +++ b/searchlib/src/tests/postinglistbm/stress_runner.cpp @@ -33,7 +33,7 @@ private: StressMaster &operator=(const StressMaster &); - search::Rand48 &_rnd; + vespalib::Rand48 &_rnd; uint32_t _numDocs; std::vector<std::string> _postingTypes; StressRunner::OperatorType _operatorType; @@ -62,7 +62,7 @@ private: std::vector<Task> _tasks; public: - StressMaster(search::Rand48 &rnd, + StressMaster(vespalib::Rand48 &rnd, FakeWordSet &wordSet, const std::vector<std::string> &postingType, StressRunner::OperatorType operatorType, @@ -129,7 +129,7 @@ public: }; -StressMaster::StressMaster(search::Rand48 &rnd, +StressMaster::StressMaster(vespalib::Rand48 &rnd, FakeWordSet &wordSet, const std::vector<std::string> &postingTypes, StressRunner::OperatorType operatorType, @@ -420,7 +420,7 @@ OrStressWorker::run_task(const FakePosting& f1, const FakePosting& f2, uint32_t } void -StressRunner::run(search::Rand48 &rnd, +StressRunner::run(vespalib::Rand48 &rnd, FakeWordSet &wordSet, const std::vector<std::string> &postingTypes, OperatorType operatorType, diff --git a/searchlib/src/tests/postinglistbm/stress_runner.h b/searchlib/src/tests/postinglistbm/stress_runner.h index d4974bc969e..249645f70aa 100644 --- a/searchlib/src/tests/postinglistbm/stress_runner.h +++ b/searchlib/src/tests/postinglistbm/stress_runner.h @@ -4,12 +4,9 @@ #include <string> #include <vector> -namespace search { -class Rand48; +namespace vespalib { class Rand48; } -namespace fakedata { class FakeWordSet; } - -} +namespace search::fakedata { class FakeWordSet; } namespace postinglistbm { @@ -21,7 +18,7 @@ public: Or }; - static void run(search::Rand48 &rnd, + static void run(vespalib::Rand48 &rnd, search::fakedata::FakeWordSet &wordSet, const std::vector<std::string> &postingTypes, OperatorType operatorType, diff --git a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp index a8aab279a98..1cf39183206 100644 --- a/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp +++ b/searchlib/src/tests/queryeval/sparse_vector_benchmark/sparse_vector_benchmark_test.cpp @@ -9,7 +9,7 @@ #include <vespa/searchlib/queryeval/andnotsearch.h> #include <vespa/searchlib/queryeval/andsearch.h> #include <vespa/searchlib/queryeval/dot_product_search.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/searchlib/queryeval/orsearch.h> #include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/queryeval/wand/weak_and_search.h> diff --git a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp index b9e27d413f3..7aa78fbe06b 100644 --- a/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp +++ b/searchlib/src/tests/tensor/hnsw_saver/hnsw_save_load_test.cpp @@ -3,7 +3,7 @@ #include <vespa/searchlib/tensor/hnsw_graph.h> #include <vespa/searchlib/tensor/hnsw_index_saver.h> #include <vespa/searchlib/tensor/hnsw_index_loader.h> -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/searchlib/util/fileutil.h> #include <vespa/vespalib/gtest/gtest.h> #include <vector> diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp index 74e48081e17..81a3006dbff 100644 --- a/searchlib/src/tests/transactionlogstress/translogstress.cpp +++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp @@ -3,7 +3,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/searchlib/transactionlog/translogclient.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/searchlib/util/runnable.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/fastos/app.h> @@ -37,7 +37,7 @@ using Visitor = TransLogClient::Visitor; class BufferGenerator { private: - Rand48 _rnd; + vespalib::Rand48 _rnd; uint32_t _minStrLen; uint32_t _maxStrLen; @@ -71,7 +71,7 @@ BufferGenerator::getRandomBuffer() class EntryGenerator { private: - Rand48 _rnd; + vespalib::Rand48 _rnd; long _baseSeed; BufferGenerator _bufferGenerator; const std::vector<nbostream> * _buffers; @@ -93,7 +93,7 @@ public: }; SerialNum getRandomSerialNum(SerialNum begin, SerialNum end); Packet::Entry getRandomEntry(SerialNum num); - Rand48 & getRnd() { return _rnd; } + vespalib::Rand48 & getRnd() { return _rnd; } void setBuffers(const std::vector<nbostream> & buffers) { _buffers = &buffers; } diff --git a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp index 33afa67e660..04addfa19f6 100644 --- a/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp +++ b/searchlib/src/tests/util/bufferwriter/bufferwriter_test.cpp @@ -2,9 +2,9 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/searchlib/util/drainingbufferwriter.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> namespace search { @@ -130,7 +130,7 @@ TEST("Test that bufferwriter passes on written data") const size_t drainerBufferSize = DrainingBufferWriter::BUFFER_SIZE; EXPECT_GREATER(mysize * sizeof(int), drainerBufferSize); a.reserve(mysize); - search::Rand48 rnd; + vespalib::Rand48 rnd; for (uint32_t i = 0; i < mysize; ++i) { a.emplace_back(rnd.lrand48()); } diff --git a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h index c4a26ff79c5..a6052f5f6b6 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h +++ b/searchlib/src/vespa/searchlib/attribute/attributefilebufferwriter.h @@ -3,7 +3,7 @@ #pragma once #include "iattributefilewriter.h" -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp index cda2bea9fb1..cd2a423782e 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_dictionary.cpp @@ -9,7 +9,7 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/datastore/datastore.hpp> #include <vespa/vespalib/datastore/unique_store_dictionary.hpp> -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.attribute.enum_store_dictionary"); diff --git a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp index cf394623abd..e0ce0ef155e 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattributesaver.cpp @@ -3,7 +3,7 @@ #include "enumattributesaver.h" #include "i_enum_store_dictionary.h" #include "iattributesavetarget.h" -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/datastore/unique_store_enumerator.hpp> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index e39ecc17aac..b76cf0c4192 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -17,7 +17,7 @@ #include <vespa/vespalib/datastore/unique_store.hpp> #include <vespa/vespalib/datastore/unique_store_string_allocator.hpp> #include <vespa/vespalib/util/array.hpp> -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/searchcommon/common/compaction_strategy.h> namespace search { diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp index 51821389a92..fe19bf236aa 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericattributesaver.cpp @@ -3,7 +3,7 @@ #include "multinumericattributesaver.h" #include "multivalueattributesaverutils.h" #include "multivalue.h" -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> using vespalib::GenerationHandler; using search::multivalueattributesaver::CountWriter; diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index eafa5bf0e1f..003fb9fa6b7 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -9,7 +9,7 @@ #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/regexp.h> #include <vespa/searchlib/query/query_term_ucs4.h> diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp index 697dbcfd27b..16acf70eb59 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "multivalueattributesaverutils.h" -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> namespace search::multivalueattributesaver { diff --git a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h index 213fd4c4777..e34f43deb0b 100644 --- a/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h +++ b/searchlib/src/vespa/searchlib/attribute/multivalueattributesaverutils.h @@ -3,7 +3,7 @@ #pragma once #include "iattributesavetarget.h" -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/arrayref.h> namespace search::multivalueattributesaver { diff --git a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp index e0a6ea193e0..4385911d0b3 100644 --- a/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/reference_attribute_saver.cpp @@ -2,7 +2,7 @@ #include "reference_attribute_saver.h" #include <vespa/vespalib/util/array.hpp> -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include "iattributesavetarget.h" diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp index afa893458b4..f3552db7c05 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattributesaver.cpp @@ -2,7 +2,7 @@ #include "singleenumattributesaver.h" #include <vespa/vespalib/util/array.hpp> -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include "iattributesavetarget.h" diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp index 406cbbbe447..e362ecff6cd 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp @@ -9,7 +9,7 @@ #include <vespa/fastlib/io/bufferedfile.h> #include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/regexp.h> #include <vespa/searchlib/query/query_term_ucs4.h> diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp index 83776d22fee..215ee6dabbe 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/vespalib/data/databuffer.h> +#include <vespa/vespalib/util/arrayref.h> namespace search::bitcompression { @@ -263,6 +264,17 @@ writeBits(const uint64_t *bits, uint32_t bitOffset, uint32_t bitLength) } } +template <bool bigEndian> +void +FeatureEncodeContext<bigEndian>::writeBytes(vespalib::ConstArrayRef<char> buf) +{ + for (unsigned char c : buf) { + writeBits(c, 8); + if (__builtin_expect(_valI >= _valE, false)) { + _writeContext->writeComprBuffer(false); + } + } +} template <bool bigEndian> void diff --git a/searchlib/src/vespa/searchlib/bitcompression/compression.h b/searchlib/src/vespa/searchlib/bitcompression/compression.h index 350932263c3..56dcee2751f 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/compression.h +++ b/searchlib/src/vespa/searchlib/bitcompression/compression.h @@ -7,7 +7,12 @@ #include <vespa/vespalib/stllike/string.h> #include <cassert> -namespace vespalib { class GenericHeader; } +namespace vespalib { + +class GenericHeader; +template <typename T> class ConstArrayRef; + +} namespace search::index { class DocIdAndFeatures; } @@ -1652,6 +1657,9 @@ public: writeBits(const uint64_t *bits, uint32_t bitOffset, uint32_t bitLength); void + writeBytes(vespalib::ConstArrayRef<char> buf); + + void writeString(vespalib::stringref buf); virtual void diff --git a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp index 33f18d83af5..4f947c6c55d 100644 --- a/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp +++ b/searchlib/src/vespa/searchlib/bitcompression/pagedict4.cpp @@ -5,6 +5,7 @@ #include "countcompression.h" #include <vespa/searchlib/index/postinglistcounts.h> #include <vespa/searchlib/index/dictionaryfile.h> +#include <vespa/vespalib/util/arrayref.h> #include <sstream> #include <vespa/log/log.h> @@ -379,14 +380,7 @@ PageDict4SPWriter::flushPage() _prevL3Size - wordsSize * 8; e.padBits(padding); if (wordsSize > 0) { - // Pad with 7 NUL bytes to silence testing tools. - _words.reserve(_words.size() + 7); - memset(&*_words.end(), '\0', 7); - const char *wordsBufX = static_cast<const char *>(&_words[0]); - size_t wordsBufXOff = reinterpret_cast<unsigned long>(wordsBufX) & 7; - const uint64_t *wordsBuf = reinterpret_cast<const uint64_t *> - (wordsBufX - wordsBufXOff); - e.writeBits(wordsBuf, 8 * wordsBufXOff, wordsSize * 8); + e.writeBytes(vespalib::ConstArrayRef<char>(&_words[0], wordsSize)); } assert((e.getWriteOffset() & (getPageBitSize() - 1)) == 0); _l6Word = _l3Word; @@ -701,14 +695,7 @@ PageDict4PWriter::flushPage() _countsSize - _countsWordOffset * 8; e.padBits(padding); if (_countsWordOffset > 0) { - // Pad with 7 NUL bytes to silence testing tools. - _words.reserve(_words.size() + 7); - memset(&*_words.end(), '\0', 7); - const char *wordsBufX = static_cast<const char *>(&_words[0]); - size_t wordsBufXOff = reinterpret_cast<unsigned long>(wordsBufX) & 7; - const uint64_t *wordsBuf = reinterpret_cast<const uint64_t *> - (wordsBufX - wordsBufXOff); - e.writeBits(wordsBuf, 8 * wordsBufXOff, _countsWordOffset * 8); + e.writeBytes(vespalib::ConstArrayRef(&_words[0], _countsWordOffset)); } assert((e.getWriteOffset() & (getPageBitSize() - 1)) == 0); _l3Word = _pendingCountsWord; diff --git a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp index 9ca49107079..fd722d0676f 100644 --- a/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp +++ b/searchlib/src/vespa/searchlib/expression/documentfieldnode.cpp @@ -261,7 +261,7 @@ DocumentFieldNode::visitMembers(vespalib::ObjectVisitor &visitor) const class String2ResultNode : public ResultNode { public: - String2ResultNode(const vespalib::string & s) : _s(s) { } + String2ResultNode(vespalib::stringref s) : _s(s) { } int64_t onGetInteger(size_t index) const override { (void) index; return strtoul(_s.c_str(), NULL, 0); } double onGetFloat(size_t index) const override { (void) index; return vespalib::locale::c::strtod(_s.c_str(), NULL); } ConstBufferRef onGetString(size_t index, BufferRef buf) const override { (void) index; (void) buf; return ConstBufferRef(_s.c_str(), _s.size()); } @@ -269,7 +269,7 @@ private: String2ResultNode * clone() const override { return new String2ResultNode(_s); } void set(const ResultNode&) override; size_t hash() const override { return 0; } - const vespalib::string & _s; + vespalib::string _s; }; void String2ResultNode::set(const ResultNode&) diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.h b/searchlib/src/vespa/searchlib/features/randomfeature.h index 8c94b71071f..3a93e77d4d0 100644 --- a/searchlib/src/vespa/searchlib/features/randomfeature.h +++ b/searchlib/src/vespa/searchlib/features/randomfeature.h @@ -3,7 +3,7 @@ #pragma once #include <vespa/searchlib/fef/blueprint.h> -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> namespace search::features { @@ -12,8 +12,8 @@ namespace search::features { **/ class RandomExecutor : public search::fef::FeatureExecutor { private: - Rand48 _rnd; // seeded once per query - Rand48 _matchRnd; // seeded once per match + vespalib::Rand48 _rnd; // seeded once per query + vespalib::Rand48 _matchRnd; // seeded once per match uint64_t _matchSeed; public: diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index 68ce0c1bb00..c9ed4039655 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -59,6 +59,21 @@ TensorReader::is_present() { return true; } +bool +can_use_index_save_file(const search::attribute::Config &config, const search::attribute::AttributeHeader &header) +{ + if (!config.hnsw_index_params().has_value() || !header.get_hnsw_index_params().has_value()) { + return false; + } + const auto &config_params = config.hnsw_index_params().value(); + const auto &header_params = header.get_hnsw_index_params().value(); + if ((config_params.max_links_per_node() != header_params.max_links_per_node()) || + (config_params.distance_metric() != header_params.distance_metric())) { + return false; + } + return true; +} + } void @@ -152,6 +167,7 @@ DenseTensorAttribute::onLoad() return false; } bool has_index_file = LoadUtils::file_exists(*this, DenseTensorAttributeSaver::index_file_suffix()); + bool use_index_file = has_index_file && _index && can_use_index_save_file(getConfig(), search::attribute::AttributeHeader::extractTags(tensorReader.getDatHeader())); setCreateSerialNum(tensorReader.getCreateSerialNum()); assert(tensorReader.getVersion() == DENSE_TENSOR_ATTRIBUTE_VERSION); @@ -165,7 +181,7 @@ DenseTensorAttribute::onLoad() auto raw = _denseTensorStore.allocRawBuffer(); tensorReader.readTensor(raw.data, _denseTensorStore.getBufSize()); _refVector.push_back(raw.ref); - if (_index && !has_index_file) { + if (_index && !use_index_file) { // This ensures that get_vector() (via getTensor()) is able to find the newly added tensor. setCommittedDocIdLimit(lid + 1); _index->add_document(lid); @@ -176,7 +192,7 @@ DenseTensorAttribute::onLoad() } setNumDocs(numDocs); setCommittedDocIdLimit(numDocs); - if (_index && has_index_file) { + if (_index && use_index_file) { auto buffer = LoadUtils::loadFile(*this, DenseTensorAttributeSaver::index_file_suffix()); if (!_index->load(*buffer)) { return false; diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp index fd8d6162f01..362e1b45266 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute_saver.cpp @@ -3,7 +3,7 @@ #include "dense_tensor_attribute_saver.h" #include "dense_tensor_store.h" #include "nearest_neighbor_index_saver.h" -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/searchlib/attribute/iattributesavetarget.h> using vespalib::GenerationHandler; diff --git a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp index 2f0bb731aa4..81ec3a5218e 100644 --- a/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/generic_tensor_attribute_saver.cpp @@ -2,7 +2,7 @@ #include "generic_tensor_attribute_saver.h" #include "generic_tensor_store.h" -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vespa/searchlib/attribute/iattributesavetarget.h> using vespalib::GenerationHandler; diff --git a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp index acff30f8cbf..46a988d575e 100644 --- a/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp +++ b/searchlib/src/vespa/searchlib/tensor/hnsw_index_saver.cpp @@ -2,7 +2,7 @@ #include "hnsw_index_saver.h" #include "hnsw_graph.h" -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> namespace search::tensor { diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp index 19080acaad3..4681861dd73 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.cpp @@ -31,7 +31,7 @@ namespace fakedata static void fillbitset(search::BitVector *bitvector, unsigned int size, - search::Rand48 &rnd) + vespalib::Rand48 &rnd) { unsigned int range; unsigned int idx; @@ -70,7 +70,7 @@ static void fillcorrelatedbitset(search::BitVector &bitvector, unsigned int size, const FakeWord &otherword, - search::Rand48 &rnd) + vespalib::Rand48 &rnd) { const FakeWord::DocWordFeatureList &opostings = otherword._postings; @@ -145,7 +145,7 @@ FakeWord::FakeWord(uint32_t docIdLimit, for (uint32_t docId : docIds) { bitmap->setBit(docId); } - search::Rand48 rnd; + vespalib::Rand48 rnd; fakeup(*bitmap, rnd, _postings, _wordPosFeatures); } @@ -153,7 +153,7 @@ FakeWord::FakeWord(uint32_t docIdLimit, uint32_t wordDocs, uint32_t tempWordDocs, const std::string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const PosOccFieldsParams &fieldsParams, uint32_t packedIndex) : _postings(), @@ -181,7 +181,7 @@ FakeWord::FakeWord(uint32_t docIdLimit, const std::string &name, const FakeWord &otherWord, size_t overlapDocs, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const PosOccFieldsParams &fieldsParams, uint32_t packedIndex) : _postings(), @@ -211,7 +211,7 @@ FakeWord::~FakeWord() void FakeWord::fakeup(search::BitVector &bitmap, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, DocWordFeatureList &postings, DocWordPosFeatureList &wordPosFeatures) { @@ -301,7 +301,7 @@ FakeWord::fakeup(search::BitVector &bitmap, void -FakeWord::fakeupTemps(search::Rand48 &rnd, +FakeWord::fakeupTemps(vespalib::Rand48 &rnd, uint32_t docIdLimit, uint32_t tempWordDocs) { @@ -315,7 +315,7 @@ FakeWord::fakeupTemps(search::Rand48 &rnd, } void -FakeWord::setupRandomizer(search::Rand48 &rnd) +FakeWord::setupRandomizer(vespalib::Rand48 &rnd) { typedef DocWordFeatureList DWFL; Randomizer randomAdd; diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h index 30f960c2d4c..ffe9e2e54d0 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeword.h @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vector> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/bitcompression/compression.h> @@ -156,16 +156,16 @@ public: void fakeup(search::BitVector &bitmap, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, DocWordFeatureList &postings, DocWordPosFeatureList &wordPosFeatures); void - fakeupTemps(search::Rand48 &rnd, + fakeupTemps(vespalib::Rand48 &rnd, uint32_t docIdLimit, uint32_t tempWordDocs); - void setupRandomizer(search::Rand48 &rnd); + void setupRandomizer(vespalib::Rand48 &rnd); const DocWordFeature & getDocWordFeature(const Randomizer &r) const @@ -220,7 +220,7 @@ public: uint32_t wordDocs, uint32_t tempWordDocs, const std::string &name, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const PosOccFieldsParams &fieldsParams, uint32_t packedIndex); @@ -230,7 +230,7 @@ public: const std::string &name, const FakeWord &otherWord, size_t overlapDocs, - search::Rand48 &rnd, + vespalib::Rand48 &rnd, const PosOccFieldsParams &fieldsParams, uint32_t packedIndex); diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp index ba691e507fa..6bb39598fab 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.cpp @@ -77,7 +77,7 @@ FakeWordSet::setupParams(bool hasElements, } void -FakeWordSet::setupWords(search::Rand48 &rnd, +FakeWordSet::setupWords(vespalib::Rand48 &rnd, uint32_t numDocs, uint32_t commonDocFreq, uint32_t numWordsPerWordClass) @@ -86,7 +86,7 @@ FakeWordSet::setupWords(search::Rand48 &rnd, } void -FakeWordSet::setupWords(search::Rand48 &rnd, +FakeWordSet::setupWords(vespalib::Rand48 &rnd, uint32_t numDocs, uint32_t commonDocFreq, uint32_t mediumDocFreq, diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h index 3dcf47cf8c8..de7efef1517 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakewordset.h @@ -6,7 +6,7 @@ #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/bitcompression/posocc_fields_params.h> -namespace search { class Rand48; } +namespace vespalib { class Rand48; } namespace search::fakedata { @@ -46,12 +46,12 @@ public: void setupParams(bool hasElements, bool hasElementWeights); - void setupWords(search::Rand48 &rnd, + void setupWords(vespalib::Rand48 &rnd, uint32_t numDocs, uint32_t commonDocFreq, uint32_t numWordsPerWordClass); - void setupWords(search::Rand48 &rnd, + void setupWords(vespalib::Rand48 &rnd, uint32_t numDocs, uint32_t commonDocFreq, uint32_t mediumDocFreq, diff --git a/searchlib/src/vespa/searchlib/util/CMakeLists.txt b/searchlib/src/vespa/searchlib/util/CMakeLists.txt index ac5fc5f54fc..68dee99339f 100644 --- a/searchlib/src/vespa/searchlib/util/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/util/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_util OBJECT SOURCES + bufferwriter.cpp comprbuffer.cpp comprfile.cpp dirtraverse.cpp diff --git a/vespalib/src/vespa/vespalib/util/bufferwriter.cpp b/searchlib/src/vespa/searchlib/util/bufferwriter.cpp index 6e57d6f58d4..6e57d6f58d4 100644 --- a/vespalib/src/vespa/vespalib/util/bufferwriter.cpp +++ b/searchlib/src/vespa/searchlib/util/bufferwriter.cpp diff --git a/vespalib/src/vespa/vespalib/util/bufferwriter.h b/searchlib/src/vespa/searchlib/util/bufferwriter.h index 3da6e3f8030..3da6e3f8030 100644 --- a/vespalib/src/vespa/vespalib/util/bufferwriter.h +++ b/searchlib/src/vespa/searchlib/util/bufferwriter.h diff --git a/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h b/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h index 38d8337fab5..f0a58b0d25a 100644 --- a/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h +++ b/searchlib/src/vespa/searchlib/util/drainingbufferwriter.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vespalib/util/bufferwriter.h> +#include <vespa/searchlib/util/bufferwriter.h> #include <vector> #include <cstdint> diff --git a/searchlib/src/vespa/searchlib/util/rand48.h b/searchlib/src/vespa/searchlib/util/rand48.h deleted file mode 100644 index 8ef529493c1..00000000000 --- a/searchlib/src/vespa/searchlib/util/rand48.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <cstdlib> -#include <cstdint> - -namespace search { - -/* - * Simple random generator based on lrand48() spec. - */ -class Rand48 -{ -private: - uint64_t _state; -public: - void - srand48(long seed) - { - _state = ((static_cast<uint64_t>(seed & 0xffffffffu)) << 16) + 0x330e; - } - - Rand48() - : _state(0) - { - srand48(0x1234abcd); - }; - void iterate() { - _state = (UINT64_C(0x5DEECE66D) * _state + 0xb) & - UINT64_C(0xFFFFFFFFFFFF); - } - /* - * Return value from 0 to 2^31 - 1 - */ - long lrand48() { - iterate(); - return static_cast<long>(_state >> 17); - } -}; - -} // namespace search - diff --git a/searchlib/src/vespa/searchlib/util/random_normal.h b/searchlib/src/vespa/searchlib/util/random_normal.h index 74596066312..c74935de02f 100644 --- a/searchlib/src/vespa/searchlib/util/random_normal.h +++ b/searchlib/src/vespa/searchlib/util/random_normal.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <cmath> namespace search { @@ -14,7 +14,7 @@ namespace search { class RandomNormal { private: - Rand48 _rnd; + vespalib::Rand48 _rnd; double _mean; double _stddev; diff --git a/searchlib/src/vespa/searchlib/util/randomgenerator.h b/searchlib/src/vespa/searchlib/util/randomgenerator.h index 40305a1d02d..ac23b387740 100644 --- a/searchlib/src/vespa/searchlib/util/randomgenerator.h +++ b/searchlib/src/vespa/searchlib/util/randomgenerator.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/searchlib/util/rand48.h> +#include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/stllike/string.h> #include <vector> #include <cassert> @@ -11,7 +11,7 @@ namespace search { class RandomGenerator { private: - Rand48 _rnd; + vespalib::Rand48 _rnd; public: RandomGenerator() : _rnd() {} diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java index 28854c59b2c..9527d50f339 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/ConfigFileBasedTlsContext.java @@ -1,32 +1,20 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.security.tls; -import com.yahoo.security.KeyStoreBuilder; -import com.yahoo.security.KeyStoreType; -import com.yahoo.security.KeyUtils; import com.yahoo.security.SslContextBuilder; -import com.yahoo.security.X509CertificateUtils; import com.yahoo.security.tls.authz.PeerAuthorizerTrustManager; import com.yahoo.security.tls.policy.AuthorizedPeers; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; import javax.net.ssl.SSLParameters; -import java.io.IOException; -import java.io.UncheckedIOException; import java.lang.ref.WeakReference; import java.nio.file.Path; -import java.security.KeyStore; -import java.time.Duration; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; /** * A {@link TlsContext} that uses the tls configuration specified in the transport security options file. @@ -36,12 +24,26 @@ import java.util.logging.Logger; */ public class ConfigFileBasedTlsContext implements TlsContext { - private static final Duration UPDATE_PERIOD = Duration.ofHours(1); + private final TlsContext tlsContext; + private TlsManager tlsManager; - private static final Logger log = Logger.getLogger(ConfigFileBasedTlsContext.class.getName()); + static private final Map<Path, WeakReference<TlsManager>> trustManagers = new HashMap<>(); - private final TlsContext tlsContext; - private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ReloaderThreadFactory()); + private static TlsManager getOrCreateTrustManager(Path tlsOptionsConfigFile) { + synchronized (trustManagers) { + WeakReference<TlsManager> tlsRef = trustManagers.get(tlsOptionsConfigFile); + TlsManager tlsManager = null; + if (tlsRef != null) { + tlsManager = tlsRef.get(); + } + if (tlsManager == null) { + tlsManager = new TlsManager(tlsOptionsConfigFile); + trustManagers.put(tlsOptionsConfigFile, new WeakReference<>(tlsManager)); + } + tlsManager.addRef(); + return tlsManager; + } + } public ConfigFileBasedTlsContext(Path tlsOptionsConfigFile, AuthorizationMode mode) { this(tlsOptionsConfigFile, mode, PeerAuthentication.NEED); @@ -52,54 +54,22 @@ public class ConfigFileBasedTlsContext implements TlsContext { * the TLS peer authentication is enforced at a higher protocol or application layer (e.g with {@link PeerAuthentication#WANT}). */ public ConfigFileBasedTlsContext(Path tlsOptionsConfigFile, AuthorizationMode mode, PeerAuthentication peerAuthentication) { - TransportSecurityOptions options = TransportSecurityOptions.fromJsonFile(tlsOptionsConfigFile); - MutableX509TrustManager trustManager = new MutableX509TrustManager(); - MutableX509KeyManager keyManager = new MutableX509KeyManager(); - reloadTrustManager(options, trustManager); - reloadKeyManager(options, keyManager); - this.tlsContext = createDefaultTlsContext(options, mode, trustManager, keyManager, peerAuthentication); - this.scheduler.scheduleAtFixedRate(new CryptoMaterialReloader(tlsOptionsConfigFile, scheduler, trustManager, keyManager), - UPDATE_PERIOD.getSeconds()/*initial delay*/, - UPDATE_PERIOD.getSeconds(), - TimeUnit.SECONDS); + tlsManager = getOrCreateTrustManager(tlsOptionsConfigFile); + this.tlsContext = createDefaultTlsContext(tlsManager.getOptions(), mode, tlsManager.getTrustManager(), tlsManager.getKeyManager(), peerAuthentication); } - private static void reloadTrustManager(TransportSecurityOptions options, MutableX509TrustManager trustManager) { - if (options.getCaCertificatesFile().isPresent()) { - trustManager.updateTruststore(loadTruststore(options.getCaCertificatesFile().get())); - } else { - trustManager.useDefaultTruststore(); - } - } - - private static void reloadKeyManager(TransportSecurityOptions options, MutableX509KeyManager keyManager) { - if (options.getPrivateKeyFile().isPresent() && options.getCertificatesFile().isPresent()) { - keyManager.updateKeystore(loadKeystore(options.getPrivateKeyFile().get(), options.getCertificatesFile().get()), new char[0]); - } else { - keyManager.useDefaultKeystore(); - } - } - - private static KeyStore loadTruststore(Path caCertificateFile) { - try { - return KeyStoreBuilder.withType(KeyStoreType.PKCS12) - .withCertificateEntries("cert", X509CertificateUtils.certificateListFromPem(com.yahoo.vespa.jdk8compat.Files.readString(caCertificateFile))) - .build(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private static KeyStore loadKeystore(Path privateKeyFile, Path certificatesFile) { - try { - return KeyStoreBuilder.withType(KeyStoreType.PKCS12) - .withKeyEntry( - "default", - KeyUtils.fromPemEncodedPrivateKey(com.yahoo.vespa.jdk8compat.Files.readString(privateKeyFile)), - X509CertificateUtils.certificateListFromPem(com.yahoo.vespa.jdk8compat.Files.readString(certificatesFile))) - .build(); - } catch (IOException e) { - throw new UncheckedIOException(e); + // Wrapped methods from TlsContext + @Override public SSLContext context() { return tlsContext.context(); } + @Override public SSLParameters parameters() { return tlsContext.parameters(); } + @Override public SSLEngine createSslEngine() { return tlsContext.createSslEngine(); } + @Override public SSLEngine createSslEngine(String peerHost, int peerPort) { return tlsContext.createSslEngine(peerHost, peerPort); } + @Override public void close() { + synchronized (trustManagers) { + int references = tlsManager.subRef(); + if (references == 0) { + tlsManager.close(); + trustManagers.remove(tlsManager.getTlsConfigFile()); + } } } @@ -122,73 +92,4 @@ public class ConfigFileBasedTlsContext implements TlsContext { return new DefaultTlsContext(sslContext, ciphers, peerAuthentication); } - // Wrapped methods from TlsContext - @Override public SSLContext context() { return tlsContext.context(); } - @Override public SSLParameters parameters() { return tlsContext.parameters(); } - @Override public SSLEngine createSslEngine() { return tlsContext.createSslEngine(); } - @Override public SSLEngine createSslEngine(String peerHost, int peerPort) { return tlsContext.createSslEngine(peerHost, peerPort); } - - @Override - public void close() { - try { - scheduler.shutdownNow(); - if (!scheduler.awaitTermination(10, TimeUnit.SECONDS)) { - throw new RuntimeException("Unable to shutdown executor before timeout"); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - // Note: no reference to outer class (directly or indirectly) to ensure trust/key managers are eventually GCed once - // there are no more use of the outer class and the underlying SSLContext - private static class CryptoMaterialReloader implements Runnable { - - final Path tlsOptionsConfigFile; - final ScheduledExecutorService scheduler; - final WeakReference<MutableX509TrustManager> trustManager; - final WeakReference<MutableX509KeyManager> keyManager; - - CryptoMaterialReloader(Path tlsOptionsConfigFile, - ScheduledExecutorService scheduler, - MutableX509TrustManager trustManager, - MutableX509KeyManager keyManager) { - this.tlsOptionsConfigFile = tlsOptionsConfigFile; - this.scheduler = scheduler; - this.trustManager = new WeakReference<>(trustManager); - this.keyManager = new WeakReference<>(keyManager); - } - - @Override - public void run() { - try { - MutableX509TrustManager trustManager = this.trustManager.get(); - MutableX509KeyManager keyManager = this.keyManager.get(); - if (trustManager == null && keyManager == null) { - scheduler.shutdown(); - return; - } - TransportSecurityOptions options = TransportSecurityOptions.fromJsonFile(tlsOptionsConfigFile); - if (trustManager != null) { - reloadTrustManager(options, trustManager); - } - if (keyManager != null) { - reloadKeyManager(options, keyManager); - } - } catch (Throwable t) { - log.log(Level.SEVERE, String.format("Failed to reload crypto material (path='%s'): %s", tlsOptionsConfigFile, t.getMessage()), t); - } - } - } - - // Static class to ensure no reference to outer class is contained - private static class ReloaderThreadFactory implements ThreadFactory { - @Override - public Thread newThread(Runnable r) { - Thread thread = new Thread(r, "tls-context-reloader"); - thread.setDaemon(true); - return thread; - } - } - } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TlsManager.java b/security-utils/src/main/java/com/yahoo/security/tls/TlsManager.java new file mode 100644 index 00000000000..c2286c3e8ac --- /dev/null +++ b/security-utils/src/main/java/com/yahoo/security/tls/TlsManager.java @@ -0,0 +1,154 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.security.tls; + +import com.yahoo.security.KeyStoreBuilder; +import com.yahoo.security.KeyStoreType; +import com.yahoo.security.KeyUtils; +import com.yahoo.security.X509CertificateUtils; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.lang.ref.WeakReference; +import java.nio.file.Path; +import java.security.KeyStore; +import java.time.Duration; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; + +class TlsManager { + private static final Logger log = Logger.getLogger(TlsManager.class.getName()); + private static final Duration UPDATE_PERIOD = Duration.ofHours(1); + + private final Path tlsConfigFile; + private final MutableX509TrustManager trustManager; + private final MutableX509KeyManager keyManager; + private final ScheduledExecutorService scheduler; + private TransportSecurityOptions options; + private final AtomicInteger references = new AtomicInteger(0); + + private static void reloadTrustManager(TransportSecurityOptions options, MutableX509TrustManager trustManager) { + if (options.getCaCertificatesFile().isPresent()) { + trustManager.updateTruststore(loadTruststore(options.getCaCertificatesFile().get())); + } else { + trustManager.useDefaultTruststore(); + } + } + + private static void reloadKeyManager(TransportSecurityOptions options, MutableX509KeyManager keyManager) { + if (options.getPrivateKeyFile().isPresent() && options.getCertificatesFile().isPresent()) { + keyManager.updateKeystore(loadKeystore(options.getPrivateKeyFile().get(), options.getCertificatesFile().get()), new char[0]); + } else { + keyManager.useDefaultKeystore(); + } + } + private static KeyStore loadTruststore(Path caCertificateFile) { + try { + return KeyStoreBuilder.withType(KeyStoreType.PKCS12) + .withCertificateEntries("cert", X509CertificateUtils.certificateListFromPem(com.yahoo.vespa.jdk8compat.Files.readString(caCertificateFile))) + .build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static KeyStore loadKeystore(Path privateKeyFile, Path certificatesFile) { + try { + return KeyStoreBuilder.withType(KeyStoreType.PKCS12) + .withKeyEntry( + "default", + KeyUtils.fromPemEncodedPrivateKey(com.yahoo.vespa.jdk8compat.Files.readString(privateKeyFile)), + X509CertificateUtils.certificateListFromPem(com.yahoo.vespa.jdk8compat.Files.readString(certificatesFile))) + .build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + // Note: no reference to outer class (directly or indirectly) to ensure trust/key managers are eventually GCed once + // there are no more use of the outer class and the underlying SSLContext + private static class CryptoMaterialReloader implements Runnable { + + private final Path tlsOptionsConfigFile; + private final ScheduledExecutorService scheduler; + private final WeakReference<TlsManager> tlsManager; + + CryptoMaterialReloader(Path tlsOptionsConfigFile, + ScheduledExecutorService scheduler, + TlsManager tlsManager) { + this.tlsOptionsConfigFile = tlsOptionsConfigFile; + this.scheduler = scheduler; + this.tlsManager = new WeakReference<>(tlsManager); + } + + @Override + public void run() { + try { + TlsManager tlsManager = this.tlsManager.get(); + if (tlsManager == null) { + // If reference count is done correctly this should not be necessary. + scheduler.shutdown(); + return; + } + TransportSecurityOptions options = TransportSecurityOptions.fromJsonFile(tlsOptionsConfigFile); + reloadTrustManager(options, tlsManager.getTrustManager()); + reloadKeyManager(options, tlsManager.getKeyManager()); + } catch (Throwable t) { + log.log(Level.SEVERE, String.format("Failed to reload crypto material (path='%s'): %s", tlsOptionsConfigFile, t.getMessage()), t); + } + } + } + + // Static class to ensure no reference to outer class is contained + private static class ReloaderThreadFactory implements ThreadFactory { + Path fileName; + ReloaderThreadFactory(Path fileName) { + this.fileName = fileName; + } + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r, "tls-context-reloader:" + fileName.toString() ); + thread.setDaemon(true); + return thread; + } + } + + TlsManager(Path tlsOptionsConfigFile) { + tlsConfigFile = tlsOptionsConfigFile; + trustManager = new MutableX509TrustManager(); + keyManager = new MutableX509KeyManager(); + options = TransportSecurityOptions.fromJsonFile(tlsOptionsConfigFile); + reloadTrustManager(options, trustManager); + reloadKeyManager(options, keyManager); + scheduler = Executors.newSingleThreadScheduledExecutor(new ReloaderThreadFactory(tlsOptionsConfigFile)); + this.scheduler.scheduleAtFixedRate(new CryptoMaterialReloader(tlsOptionsConfigFile, scheduler, this), + UPDATE_PERIOD.getSeconds()/*initial delay*/, + UPDATE_PERIOD.getSeconds(), + TimeUnit.SECONDS); + } + MutableX509TrustManager getTrustManager() { + return trustManager; + } + + MutableX509KeyManager getKeyManager() { + return keyManager; + } + + Path getTlsConfigFile() { + return tlsConfigFile; + } + + TransportSecurityOptions getOptions() { + return options; + } + + void close() { + scheduler.shutdown(); + } + + int addRef() { return references.incrementAndGet(); } + int subRef() { return references.decrementAndGet(); } +} diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java index 104f41c39ed..c9df077aeef 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java @@ -5,7 +5,7 @@ import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.service.monitor.DuperModelListener; import java.util.ArrayList; @@ -42,7 +42,7 @@ public class DuperModel { void setComplete() { if (!isComplete) { - logger.log(LogLevel.DEBUG, "All applications have been activated: duper model is complete"); + logger.log(Level.FINE, "All applications have been activated: duper model is complete"); isComplete = true; listeners.forEach(DuperModelListener::bootstrapComplete); @@ -95,7 +95,7 @@ public class DuperModel { } else { logPrefix = isComplete ? "Reactivated application " : "Rebootstrapped application "; } - logger.log(LogLevel.DEBUG, logPrefix + id.toFullString()); + logger.log(Level.FINE, logPrefix + id.toFullString()); updateHostnameVsIdMaps(applicationInfo, id); @@ -110,7 +110,7 @@ public class DuperModel { ApplicationInfo application = applicationsById.remove(applicationId); if (application != null) { - logger.log(LogLevel.DEBUG, "Removed application " + applicationId.toFullString()); + logger.log(Level.FINE, "Removed application " + applicationId.toFullString()); listeners.forEach(listener -> listener.applicationRemoved(applicationId)); } } @@ -132,7 +132,7 @@ public class DuperModel { // If an activation contains a host that is currently assigned to a // different application we will patch up our data structures to remain // internally consistent. But listeners may be fooled. - logger.log(LogLevel.WARNING, hostname + " has been reassigned from " + + logger.log(Level.WARNING, hostname + " has been reassigned from " + previousId.toFullString() + " to " + id.toFullString()); Set<HostName> previousHostnames = hostnamesById.get(previousId); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index cb1c6d2d95f..766218f8be5 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -10,7 +10,7 @@ import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.service.monitor.CriticalRegion; import com.yahoo.vespa.service.monitor.DuperModelInfraApi; @@ -102,7 +102,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi lockedRunnable(() -> { if (!superModelIsComplete) { superModelIsComplete = true; - logger.log(LogLevel.INFO, "All bootstrap tenant applications have been activated"); + logger.log(Level.INFO, "All bootstrap tenant applications have been activated"); maybeSetDuperModelAsComplete(); } }); @@ -171,7 +171,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi lockedRunnable(() -> { if (!infraApplicationsIsComplete) { infraApplicationsIsComplete = true; - logger.log(LogLevel.INFO, "All infrastructure applications have been activated"); + logger.log(Level.INFO, "All infrastructure applications have been activated"); maybeSetDuperModelAsComplete(); } }); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java index 316b810c682..cda91994443 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/CancellableImpl.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.executor; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.time.Duration; import java.util.Optional; @@ -97,7 +97,7 @@ class CancellableImpl implements Cancellable, Runnable { runlet.close(); periodicExecutionCancellation.get().run(); } catch (Throwable e) { - logger.log(LogLevel.ERROR, "Failed run of periodic execution", e); + logger.log(Level.SEVERE, "Failed run of periodic execution", e); } } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java index 1e2547cc762..39aeb94667c 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/executor/RunletExecutorImpl.java @@ -1,7 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.executor; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import java.time.Duration; import java.util.concurrent.ConcurrentHashMap; @@ -62,7 +62,7 @@ public class RunletExecutorImpl implements RunletExecutor { try { executor.awaitTermination(10, TimeUnit.MINUTES); } catch (InterruptedException e) { - logger.log(LogLevel.WARNING, "Timed out waiting for termination of executor", e); + logger.log(Level.WARNING, "Timed out waiting for termination of executor", e); } } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java index 19399c79b43..dd70c8d6992 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthClient.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.service.health; import com.fasterxml.jackson.databind.ObjectMapper; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.util.EntityUtils; @@ -68,7 +68,7 @@ public class StateV1HealthClient implements AutoCloseable { try { httpClient.close(); } catch (Exception e) { - logger.log(LogLevel.WARNING, "Failed to close CloseableHttpClient", e); + logger.log(Level.WARNING, "Failed to close CloseableHttpClient", e); } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java index e7a8d33ea14..2fa97d41284 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/slobrok/SlobrokMonitorManagerImpl.java @@ -5,7 +5,7 @@ import com.google.inject.Inject; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.jrt.slobrok.api.Mirror; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; @@ -64,7 +64,7 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager { synchronized (monitor) { SlobrokMonitor slobrokMonitor = slobrokMonitors.remove(id); if (slobrokMonitor == null) { - logger.log(LogLevel.WARNING, "Removed application " + id + + logger.log(Level.WARNING, "Removed application " + id + ", but it was never registered"); } else { slobrokMonitor.close(); @@ -149,7 +149,7 @@ public class SlobrokMonitorManagerImpl implements SlobrokApi, MonitorManager { case "storagenode": return Optional.of("storage/cluster." + configId.s()); default: - logger.log(LogLevel.DEBUG, "Unknown service type " + serviceType.s() + + logger.log(Level.FINE, "Unknown service type " + serviceType.s() + " with config id " + configId.s()); return Optional.empty(); } diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java index edb9e4820d1..b75a0529a03 100644 --- a/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/Bucket.java @@ -14,7 +14,7 @@ import java.util.logging.Logger; import com.yahoo.collections.LazyMap; import com.yahoo.collections.LazySet; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; /** * An aggregation of data which is only written to from a single thread. @@ -78,7 +78,7 @@ public class Bucket { get(entry.getKey(), entry.getValue()).merge(entry.getValue(), otherIsNewer); } } catch (IllegalArgumentException e) { - log.log(LogLevel.WARNING, "Problems merging metric " + metricName + ", possibly ignoring data."); + log.log(Level.WARNING, "Problems merging metric " + metricName + ", possibly ignoring data."); // avoid spamming the log if there are a lot of mismatching // threads malformedMetrics.add(metricName); diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java index fce5026349e..a2a7c5b94ba 100644 --- a/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/MetricManager.java @@ -11,7 +11,7 @@ import com.yahoo.concurrent.ThreadLocalDirectory; import com.yahoo.concurrent.ThreadLocalDirectory.Updater; import com.yahoo.container.di.componentgraph.Provider; import com.yahoo.metrics.ManagerConfig; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; /** * This is the coordinating class owning the executor and the top level objects @@ -32,7 +32,7 @@ public class MetricManager extends AbstractComponent implements Provider<MetricR } private MetricManager(ManagerConfig settings, Updater<Bucket, Sample> updater) { - log.log(LogLevel.CONFIG, "setting up simple metrics gathering." + + log.log(Level.CONFIG, "setting up simple metrics gathering." + " reportPeriodSeconds=" + settings.reportPeriodSeconds() + ", pointsToKeepPerMetric=" + settings.pointsToKeepPerMetric()); metricsCollection = new ThreadLocalDirectory<>(updater); diff --git a/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java b/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java index f6762434275..f757ab15022 100644 --- a/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java +++ b/simplemetrics/src/main/java/com/yahoo/metrics/simple/UntypedMetric.java @@ -5,7 +5,7 @@ import java.util.logging.Logger; import org.HdrHistogram.DoubleHistogram; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; /** * A gauge or a counter or... who knows? The class for storing a metric when the @@ -96,7 +96,7 @@ public class UntypedMetric { try { histogram.add(other.histogram); } catch (ArrayIndexOutOfBoundsException e) { - log.log(LogLevel.WARNING, "Had trouble merging histograms: " + e.getMessage()); + log.log(Level.WARNING, "Had trouble merging histograms: " + e.getMessage()); } } } diff --git a/statistics/src/main/java/com/yahoo/statistics/Value.java b/statistics/src/main/java/com/yahoo/statistics/Value.java index e9cacdebbd5..8a5dbdbd880 100644 --- a/statistics/src/main/java/com/yahoo/statistics/Value.java +++ b/statistics/src/main/java/com/yahoo/statistics/Value.java @@ -7,7 +7,7 @@ import java.util.logging.Logger; import com.yahoo.container.StatisticsConfig; import com.yahoo.container.StatisticsConfig.Values.Operations; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.event.Event; import com.yahoo.statistics.SampleSet.Sampling; @@ -360,7 +360,7 @@ public class Value extends Handle { throw new IllegalStateException("The Limits instance must be frozen."); } if (parameters.histogramId != HistogramType.REGULAR) { - log.log(LogLevel.WARNING, HISTOGRAM_TYPE_WARNING + " '" + name + "'"); + log.log(Level.WARNING, HISTOGRAM_TYPE_WARNING + " '" + name + "'"); } this.histogramId = HistogramType.REGULAR; this.histogram = parameters.limits; diff --git a/storage/src/tests/bucketdb/judyarraytest.cpp b/storage/src/tests/bucketdb/judyarraytest.cpp index 94d61107fcf..72967e8b4c0 100644 --- a/storage/src/tests/bucketdb/judyarraytest.cpp +++ b/storage/src/tests/bucketdb/judyarraytest.cpp @@ -62,7 +62,7 @@ TEST(JudyArrayTest, iterating) { JudyArray::ConstIterator cit = array.begin(); EXPECT_THAT(cit.toString(), MatchesRegex("^ConstIterator\\(Key: 3, Valp: 0x[0-9a-f]{1,16}, Val: 2\\)$")); JudyArray::Iterator it = array.end(); - EXPECT_THAT(it.toString(), MatchesRegex("^Iterator\\(Key: 0, Valp: 0\\)$")); + EXPECT_THAT(it.toString(), MatchesRegex("^Iterator\\(Key: 0, Valp: (0x)?0\\)$")); } } diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp index 05e51993f49..fb80c25bfb7 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp @@ -145,15 +145,6 @@ PersistenceProviderWrapper::get(const spi::Bucket& bucket, return _spi.get(bucket, fieldSet, id, context); } -spi::Result -PersistenceProviderWrapper::flush(const spi::Bucket& bucket, - spi::Context& context) -{ - LOG_SPI("flush(" << bucket << ")"); - CHECK_ERROR(spi::Result, FAIL_FLUSH); - return _spi.flush(bucket, context); -} - spi::CreateIteratorResult PersistenceProviderWrapper::createIterator(const spi::Bucket& bucket, const document::FieldSet& fields, diff --git a/storage/src/tests/persistence/common/persistenceproviderwrapper.h b/storage/src/tests/persistence/common/persistenceproviderwrapper.h index 511ced02118..9bd3653e8a1 100644 --- a/storage/src/tests/persistence/common/persistenceproviderwrapper.h +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.h @@ -36,7 +36,6 @@ public: FAIL_REPLACE_WITH_REMOVE = 1 << 6, FAIL_UPDATE = 1 << 7, FAIL_REVERT = 1 << 8, - FAIL_FLUSH = 1 << 9, FAIL_CREATE_ITERATOR = 1 << 10, FAIL_ITERATE = 1 << 11, FAIL_DESTROY_ITERATOR = 1 << 12, @@ -44,7 +43,7 @@ public: FAIL_SPLIT = 1 << 14, FAIL_JOIN = 1 << 15, FAIL_CREATE_BUCKET = 1 << 16, - FAIL_BUCKET_PERSISTENCE = FAIL_PUT|FAIL_REMOVE|FAIL_UPDATE|FAIL_REVERT|FAIL_FLUSH, + FAIL_BUCKET_PERSISTENCE = FAIL_PUT|FAIL_REMOVE|FAIL_UPDATE|FAIL_REVERT, FAIL_ALL_OPERATIONS = 0xffff, // TODO: add more as needed }; @@ -55,7 +54,7 @@ private: uint32_t _failureMask; public: PersistenceProviderWrapper(spi::PersistenceProvider& spi); - ~PersistenceProviderWrapper(); + ~PersistenceProviderWrapper() override; /** * Explicitly set result to anything != NONE to have all operations @@ -96,7 +95,6 @@ public: spi::GetResult get(const spi::Bucket&, const document::FieldSet&, const spi::DocumentId&, spi::Context&) const override ; - spi::Result flush(const spi::Bucket&, spi::Context&) override; spi::CreateIteratorResult createIterator(const spi::Bucket&, const document::FieldSet&, const spi::Selection&, spi::IncludedVersions versions, spi::Context&) override; diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 4576f8a08f8..2cb390fca86 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -698,19 +698,6 @@ TEST_F(FileStorManagerTest, handler_pause) { ASSERT_EQ(30, filestorHandler.getNextMessage(0, stripeId).second->getPriority()); } -namespace { - -uint64_t getPutTime(api::StorageMessage::SP& msg) -{ - if (!msg.get()) { - return (uint64_t)-1; - } - - return static_cast<api::PutCommand*>(msg.get())->getTimestamp(); -}; - -} - TEST_F(FileStorManagerTest, remap_split) { // Setup a filestorthread to test DummyStorageLink top; @@ -768,63 +755,6 @@ TEST_F(FileStorManagerTest, remap_split) { filestorHandler.dumpQueue(0)); } -TEST_F(FileStorManagerTest, handler_multi) { - // Setup a filestorthread to test - DummyStorageLink top; - DummyStorageLink *dummyManager; - top.push_back(std::unique_ptr<StorageLink>( - dummyManager = new DummyStorageLink)); - top.open(); - ForwardingMessageSender messageSender(*dummyManager); - // Since we fake time with small numbers, we need to make sure we dont - // compact them away, as they will seem to be from 1970 - - documentapi::LoadTypeSet loadTypes("raw:"); - FileStorMetrics metrics(loadTypes.getMetricLoadTypes()); - metrics.initDiskMetrics(_node->getPartitions().size(), loadTypes.getMetricLoadTypes(), 1, 1); - - FileStorHandler filestorHandler(messageSender, metrics, _node->getPartitions(), _node->getComponentRegister()); - filestorHandler.setGetNextMessageTimeout(50); - uint32_t stripeId = filestorHandler.getNextStripeId(0); - - std::string content("Here is some content which is in all documents"); - - Document::SP doc1(createDocument(content, "id:footype:testdoctype1:n=1234:bar").release()); - - Document::SP doc2(createDocument(content, "id:footype:testdoctype1:n=4567:bar").release()); - - document::BucketIdFactory factory; - document::BucketId bucket1(16, factory.getBucketId(doc1->getId()).getRawId()); - document::BucketId bucket2(16, factory.getBucketId(doc2->getId()).getRawId()); - - // Populate bucket with the given data - for (uint32_t i = 1; i < 10; i++) { - filestorHandler.schedule( - api::StorageMessage::SP(new api::PutCommand(makeDocumentBucket(bucket1), doc1, i)), 0); - filestorHandler.schedule( - api::StorageMessage::SP(new api::PutCommand(makeDocumentBucket(bucket2), doc2, i + 10)), 0); - } - - { - FileStorHandler::LockedMessage lock = filestorHandler.getNextMessage(0, stripeId); - ASSERT_EQ(1, getPutTime(lock.second)); - - lock = filestorHandler.getNextMessage(0, stripeId, lock); - ASSERT_EQ(2, getPutTime(lock.second)); - - lock = filestorHandler.getNextMessage(0, stripeId, lock); - ASSERT_EQ(3, getPutTime(lock.second)); - } - - { - FileStorHandler::LockedMessage lock = filestorHandler.getNextMessage(0, stripeId); - ASSERT_EQ(11, getPutTime(lock.second)); - - lock = filestorHandler.getNextMessage(0, stripeId, lock); - ASSERT_EQ(12, getPutTime(lock.second)); - } -} - TEST_F(FileStorManagerTest, handler_timeout) { // Setup a filestorthread to test DummyStorageLink top; diff --git a/storage/src/tests/persistence/mergehandlertest.cpp b/storage/src/tests/persistence/mergehandlertest.cpp index 0b2baab5652..7e7a76b95a8 100644 --- a/storage/src/tests/persistence/mergehandlertest.cpp +++ b/storage/src/tests/persistence/mergehandlertest.cpp @@ -626,8 +626,7 @@ MergeHandlerTest::createDummyGetBucketDiff(int timestampOffset, } TEST_F(MergeHandlerTest, spi_flush_guard) { - PersistenceProviderWrapper providerWrapper( - getPersistenceProvider()); + PersistenceProviderWrapper providerWrapper(getPersistenceProvider()); MergeHandler handler(providerWrapper, getEnv()); providerWrapper.setResult( @@ -635,8 +634,7 @@ TEST_F(MergeHandlerTest, spi_flush_guard) { setUpChain(MIDDLE); // Fail applying unrevertable remove - providerWrapper.setFailureMask( - PersistenceProviderWrapper::FAIL_REMOVE); + providerWrapper.setFailureMask(PersistenceProviderWrapper::FAIL_REMOVE); providerWrapper.clearOperationLog(); try { @@ -645,11 +643,6 @@ TEST_F(MergeHandlerTest, spi_flush_guard) { } catch (const std::runtime_error& e) { EXPECT_TRUE(std::string(e.what()).find("Failed remove") != std::string::npos); } - // Test that we always flush after applying diff locally, even when - // errors are encountered. - const std::vector<std::string>& opLog(providerWrapper.getOperationLog()); - ASSERT_FALSE(opLog.empty()); - EXPECT_EQ("flush(Bucket(0x40000000000004d2, partition 0))", opLog.back()); } TEST_F(MergeHandlerTest, bucket_not_found_in_db) { @@ -901,7 +894,6 @@ TEST_F(MergeHandlerTest, apply_bucket_diff_spi_failures) { { PersistenceProviderWrapper::FAIL_ITERATE, "iterate" }, { PersistenceProviderWrapper::FAIL_PUT, "Failed put" }, { PersistenceProviderWrapper::FAIL_REMOVE, "Failed remove" }, - { PersistenceProviderWrapper::FAIL_FLUSH, "Failed flush" }, }; typedef ExpectedExceptionSpec* ExceptionIterator; @@ -1058,7 +1050,6 @@ TEST_F(MergeHandlerTest, apply_bucket_diff_reply_spi_failures) { { PersistenceProviderWrapper::FAIL_ITERATE, "iterate" }, { PersistenceProviderWrapper::FAIL_PUT, "Failed put" }, { PersistenceProviderWrapper::FAIL_REMOVE, "Failed remove" }, - { PersistenceProviderWrapper::FAIL_FLUSH, "Failed flush" }, }; typedef ExpectedExceptionSpec* ExceptionIterator; diff --git a/storage/src/tests/persistence/persistencequeuetest.cpp b/storage/src/tests/persistence/persistencequeuetest.cpp index be276dd7f9d..3754a82e7ae 100644 --- a/storage/src/tests/persistence/persistencequeuetest.cpp +++ b/storage/src/tests/persistence/persistencequeuetest.cpp @@ -168,19 +168,4 @@ TEST_F(PersistenceQueueTest, exclusive_locked_operation_not_started_if_exclusive ASSERT_FALSE(lock1.first.get()); } -TEST_F(PersistenceQueueTest, operation_batching_not_allowed_across_different_lock_modes) { - Fixture f(*this); - - f.filestorHandler->schedule(createPut(1234, 0), _disk); - f.filestorHandler->schedule(createGet(1234), _disk); - - auto lock0 = f.filestorHandler->getNextMessage(_disk, f.stripeId); - ASSERT_TRUE(lock0.first); - ASSERT_TRUE(lock0.second); - EXPECT_EQ(api::LockingRequirements::Exclusive, lock0.first->lockingRequirements()); - - f.filestorHandler->getNextMessage(_disk, f.stripeId, lock0); - ASSERT_FALSE(lock0.second); -} - } // namespace storage diff --git a/storage/src/tests/persistence/persistencetestutils.cpp b/storage/src/tests/persistence/persistencetestutils.cpp index e32fc056413..25c0a36a7f5 100644 --- a/storage/src/tests/persistence/persistencetestutils.cpp +++ b/storage/src/tests/persistence/persistencetestutils.cpp @@ -159,7 +159,6 @@ PersistenceTestUtils::doPutOnDisk( getPersistenceProvider().put(spi::Bucket(b), timestamp, doc, context); - getPersistenceProvider().flush(b, context); return doc; } diff --git a/storage/src/tests/persistence/persistencethread_splittest.cpp b/storage/src/tests/persistence/persistencethread_splittest.cpp index ea7dce96e0c..c8318bef211 100644 --- a/storage/src/tests/persistence/persistencethread_splittest.cpp +++ b/storage/src/tests/persistence/persistencethread_splittest.cpp @@ -213,7 +213,7 @@ PersistenceThreadSplitTest::doTest(SplitCase splitCase) cmd.setMinByteSize(maxSize); cmd.setMinDocCount(maxCount); cmd.setSourceIndex(0); - MessageTracker::UP result(thread->handleSplitBucket(cmd)); + MessageTracker::UP result(thread->handleSplitBucket(cmd, context)); api::ReturnCode code(result->getResult()); EXPECT_EQ(error, code); if (!code.success()) { diff --git a/storage/src/tests/persistence/testandsettest.cpp b/storage/src/tests/persistence/testandsettest.cpp index 1507f0e8f0d..1e67e90b540 100644 --- a/storage/src/tests/persistence/testandsettest.cpp +++ b/storage/src/tests/persistence/testandsettest.cpp @@ -33,15 +33,15 @@ struct TestAndSetTest : SingleDiskPersistenceTestUtils { unique_ptr<PersistenceThread> thread; shared_ptr<document::Document> testDoc; document::DocumentId testDocId; + spi::Context context; + + TestAndSetTest() + : context(spi::LoadType(0, "default"), 0, 0) + {} void SetUp() override { SingleDiskPersistenceTestUtils::SetUp(); - spi::Context context( - spi::LoadType(0, "default"), - spi::Priority(0), - spi::Trace::TraceLevel(0)); - createBucket(BUCKET_ID); getPersistenceProvider().createBucket( makeSpiBucket(BUCKET_ID), @@ -85,7 +85,7 @@ TEST_F(TestAndSetTest, conditional_put_not_executed_on_condition_mismatch) { api::PutCommand putTwo(makeDocumentBucket(BUCKET_ID), testDoc, timestampTwo); setTestCondition(putTwo); - ASSERT_EQ(thread->handlePut(putTwo)->getResult().getResult(), + ASSERT_EQ(thread->handlePut(putTwo, context)->getResult().getResult(), api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); EXPECT_EQ(expectedDocEntryString(timestampOne, testDocId), dumpBucket(BUCKET_ID)); } @@ -105,7 +105,7 @@ TEST_F(TestAndSetTest, conditional_put_executed_on_condition_match) { api::PutCommand putTwo(makeDocumentBucket(BUCKET_ID), testDoc, timestampTwo); setTestCondition(putTwo); - ASSERT_EQ(thread->handlePut(putTwo)->getResult().getResult(), api::ReturnCode::Result::OK); + ASSERT_EQ(thread->handlePut(putTwo, context)->getResult().getResult(), api::ReturnCode::Result::OK); EXPECT_EQ(expectedDocEntryString(timestampOne, testDocId) + expectedDocEntryString(timestampTwo, testDocId), dumpBucket(BUCKET_ID)); @@ -125,7 +125,7 @@ TEST_F(TestAndSetTest, conditional_remove_not_executed_on_condition_mismatch) { api::RemoveCommand remove(makeDocumentBucket(BUCKET_ID), testDocId, timestampTwo); setTestCondition(remove); - ASSERT_EQ(thread->handleRemove(remove)->getResult().getResult(), + ASSERT_EQ(thread->handleRemove(remove, context)->getResult().getResult(), api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); EXPECT_EQ(expectedDocEntryString(timestampOne, testDocId), dumpBucket(BUCKET_ID)); @@ -145,7 +145,7 @@ TEST_F(TestAndSetTest, conditional_remove_executed_on_condition_match) { api::RemoveCommand remove(makeDocumentBucket(BUCKET_ID), testDocId, timestampTwo); setTestCondition(remove); - ASSERT_EQ(thread->handleRemove(remove)->getResult().getResult(), api::ReturnCode::Result::OK); + ASSERT_EQ(thread->handleRemove(remove, context)->getResult().getResult(), api::ReturnCode::Result::OK); EXPECT_EQ(expectedDocEntryString(timestampOne, testDocId) + expectedDocEntryString(timestampTwo, testDocId, spi::REMOVE_ENTRY), dumpBucket(BUCKET_ID)); @@ -172,7 +172,7 @@ TEST_F(TestAndSetTest, conditional_update_not_executed_on_condition_mismatch) { putTestDocument(false, timestampOne); auto updateUp = conditional_update_test(false, timestampTwo); - ASSERT_EQ(thread->handleUpdate(*updateUp)->getResult().getResult(), + ASSERT_EQ(thread->handleUpdate(*updateUp, context)->getResult().getResult(), api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); EXPECT_EQ(expectedDocEntryString(timestampOne, testDocId), dumpBucket(BUCKET_ID)); @@ -185,7 +185,7 @@ TEST_F(TestAndSetTest, conditional_update_executed_on_condition_match) { putTestDocument(true, timestampOne); auto updateUp = conditional_update_test(false, timestampTwo); - ASSERT_EQ(thread->handleUpdate(*updateUp)->getResult().getResult(), api::ReturnCode::Result::OK); + ASSERT_EQ(thread->handleUpdate(*updateUp, context)->getResult().getResult(), api::ReturnCode::Result::OK); EXPECT_EQ(expectedDocEntryString(timestampOne, testDocId) + expectedDocEntryString(timestampTwo, testDocId), dumpBucket(BUCKET_ID)); @@ -197,7 +197,7 @@ TEST_F(TestAndSetTest, conditional_update_not_executed_when_no_document_and_no_a api::Timestamp updateTimestamp = 200; auto updateUp = conditional_update_test(false, updateTimestamp); - ASSERT_EQ(thread->handleUpdate(*updateUp)->getResult().getResult(), + ASSERT_EQ(thread->handleUpdate(*updateUp, context)->getResult().getResult(), api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); EXPECT_EQ("", dumpBucket(BUCKET_ID)); } @@ -206,7 +206,7 @@ TEST_F(TestAndSetTest, conditional_update_executed_when_no_document_but_auto_cre api::Timestamp updateTimestamp = 200; auto updateUp = conditional_update_test(true, updateTimestamp); - ASSERT_EQ(thread->handleUpdate(*updateUp)->getResult().getResult(), api::ReturnCode::Result::OK); + ASSERT_EQ(thread->handleUpdate(*updateUp, context)->getResult().getResult(), api::ReturnCode::Result::OK); EXPECT_EQ(expectedDocEntryString(updateTimestamp, testDocId), dumpBucket(BUCKET_ID)); assertTestDocumentFoundAndMatchesContent(NEW_CONTENT); } @@ -218,7 +218,7 @@ TEST_F(TestAndSetTest, invalid_document_selection_should_fail) { api::PutCommand put(makeDocumentBucket(BUCKET_ID), testDoc, timestamp); put.setCondition(documentapi::TestAndSetCondition("bjarne")); - ASSERT_EQ(thread->handlePut(put)->getResult().getResult(), api::ReturnCode::Result::ILLEGAL_PARAMETERS); + ASSERT_EQ(thread->handlePut(put, context)->getResult().getResult(), api::ReturnCode::Result::ILLEGAL_PARAMETERS); EXPECT_EQ("", dumpBucket(BUCKET_ID)); } @@ -228,9 +228,9 @@ TEST_F(TestAndSetTest, conditional_put_to_non_existing_document_should_fail) { api::Timestamp timestamp = 0; api::PutCommand put(makeDocumentBucket(BUCKET_ID), testDoc, timestamp); setTestCondition(put); - thread->handlePut(put); + thread->handlePut(put, context); - ASSERT_EQ(thread->handlePut(put)->getResult().getResult(), + ASSERT_EQ(thread->handlePut(put, context)->getResult().getResult(), api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); EXPECT_EQ("", dumpBucket(BUCKET_ID)); } @@ -254,7 +254,7 @@ TestAndSetTest::createTestDocument() document::Document::SP TestAndSetTest::retrieveTestDocument() { api::GetCommand get(makeDocumentBucket(BUCKET_ID), testDocId, "[all]"); - auto tracker = thread->handleGet(get); + auto tracker = thread->handleGet(get, context); assert(tracker->getResult() == api::ReturnCode::Result::OK); auto & reply = static_cast<api::GetReply &>(*tracker->getReply()); @@ -274,7 +274,7 @@ void TestAndSetTest::putTestDocument(bool matchingHeader, api::Timestamp timesta } api::PutCommand put(makeDocumentBucket(BUCKET_ID), testDoc, timestamp); - thread->handlePut(put); + thread->handlePut(put, context); } void TestAndSetTest::assertTestDocumentFoundAndMatchesContent(const document::FieldValue & value) diff --git a/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp b/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp index 56f43db1ac5..7b75091e252 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp +++ b/storage/src/vespa/storage/distributor/maintenance/simplebucketprioritydatabase.cpp @@ -83,7 +83,7 @@ SimpleBucketPriorityDatabase::SimpleConstIteratorImpl::increment() { while (!atEnd()) { step(); - if (!currentPriorityAtEnd()) { + if (atEnd() || !currentPriorityAtEnd()) { break; } } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp index 543accc80ae..44cd8f0ab0c 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.cpp @@ -60,12 +60,6 @@ FileStorHandler::getNextMessage(uint16_t disk, uint32_t stripeId) return _impl->getNextMessage(disk, stripeId); } -FileStorHandler::LockedMessage & -FileStorHandler::getNextMessage(uint16_t disk, uint32_t stripeId, LockedMessage& lck) -{ - return _impl->getNextMessage(disk, stripeId, lck); -} - FileStorHandler::BucketLockInterface::SP FileStorHandler::lock(const document::Bucket& bucket, uint16_t disk, api::LockingRequirements lockReq) { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index db294d7c39f..af0a52b2fa0 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -121,11 +121,6 @@ public: LockedMessage getNextMessage(uint16_t disk, uint32_t stripeId); /** - * Returns the next message for the same bucket. - */ - LockedMessage & getNextMessage(uint16_t disk, uint32_t stripeId, LockedMessage& lock); - - /** * Lock a bucket. By default, each file stor thread has the locks of all * buckets in their area of responsibility. If they need to access buckets * outside of their area, they can call this to make sure the thread diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index bfd5233d017..5ad37d585ef 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -319,25 +319,6 @@ FileStorHandlerImpl::getNextStripeId(uint32_t disk) { return _diskInfo[disk].getNextStripeId(); } - -FileStorHandler::LockedMessage & -FileStorHandlerImpl::getNextMessage(uint16_t diskId, uint32_t stripeId, FileStorHandler::LockedMessage& lck) -{ - document::Bucket bucket(lck.first->getBucket()); - - LOG(spam, "Disk %d retrieving message for buffered bucket %s", diskId, bucket.getBucketId().toString().c_str()); - - assert(diskId < _diskInfo.size()); - Disk& disk(_diskInfo[diskId]); - - if (disk.isClosed()) { - lck.second.reset(); - return lck; - } - - return disk.getNextMessage(stripeId, lck); -} - bool FileStorHandlerImpl::tryHandlePause(uint16_t disk) const { @@ -977,49 +958,6 @@ FileStorHandlerImpl::Stripe::getNextMessage(uint32_t timeout, Disk & disk) return {}; // No message fetched. } -FileStorHandler::LockedMessage & -FileStorHandlerImpl::Stripe::getNextMessage(FileStorHandler::LockedMessage& lck) -{ - const document::Bucket & bucket = lck.second->getBucket(); - vespalib::MonitorGuard guard(_lock); - BucketIdx& idx = bmi::get<2>(_queue); - std::pair<BucketIdx::iterator, BucketIdx::iterator> range = idx.equal_range(bucket); - - // No more for this bucket. - if (range.first == range.second) { - lck.second.reset(); - return lck; - } - - api::StorageMessage & m(*range.first->_command); - // For now, don't allow batching of operations across lock requirement modes. - // We might relax this requirement later once we're 100% sure it can't trigger - // any unfortunate edge cases. - if (lck.first->lockingRequirements() != m.lockingRequirements()) { - lck.second.reset(); - return lck; - } - - std::chrono::milliseconds waitTime(uint64_t(range.first->_timer.stop(_metrics->averageQueueWaitingTime[m.getLoadType()]))); - - if (!messageTimedOutInQueue(m, waitTime)) { - std::shared_ptr<api::StorageMessage> msg = std::move(range.first->_command); - idx.erase(range.first); - lck.second.swap(msg); - guard.broadcast(); - } else { - std::shared_ptr<api::StorageReply> msgReply = static_cast<api::StorageCommand&>(m).makeReply(); - idx.erase(range.first); - guard.broadcast(); - guard.unlock(); - msgReply->setResult(api::ReturnCode(api::ReturnCode::TIMEOUT, "Message waited too long in storage queue")); - _messageSender.sendReply(msgReply); - - lck.second.reset(); - } - return lck; -} - FileStorHandler::LockedMessage FileStorHandlerImpl::Stripe::getMessage(vespalib::MonitorGuard & guard, PriorityIdx & idx, PriorityIdx::iterator iter) { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index da4d242a4c9..7a4f9000e82 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -129,7 +129,6 @@ public: void failOperations(const document::Bucket & bucket, const api::ReturnCode & code); FileStorHandler::LockedMessage getNextMessage(uint32_t timeout, Disk & disk); - FileStorHandler::LockedMessage & getNextMessage(FileStorHandler::LockedMessage& lock); void dumpQueue(std::ostream & os) const; void dumpActiveHtml(std::ostream & os) const; void dumpQueueHtml(std::ostream & os) const; @@ -185,9 +184,6 @@ public: FileStorHandler::LockedMessage getNextMessage(uint32_t stripeId, uint32_t timeout) { return _stripes[stripeId].getNextMessage(timeout, *this); } - FileStorHandler::LockedMessage & getNextMessage(uint32_t stripeId, FileStorHandler::LockedMessage & lck) { - return _stripes[stripeId].getNextMessage(lck); - } std::shared_ptr<FileStorHandler::BucketLockInterface> lock(const document::Bucket & bucket, api::LockingRequirements lockReq) { return stripe(bucket).lock(bucket, lockReq); @@ -253,8 +249,6 @@ public: FileStorHandler::LockedMessage getNextMessage(uint16_t disk, uint32_t stripeId); - FileStorHandler::LockedMessage & getNextMessage(uint16_t disk, uint32_t stripeId, FileStorHandler::LockedMessage& lock); - enum Operation { MOVE, SPLIT, JOIN }; void remapQueue(const RemapInfo& source, RemapInfo& target, Operation op); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index ea6f5e03a80..6eab44923f7 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -802,7 +802,7 @@ void FileStorManager::onFlush(bool downwards) LOG(debug, "Flushed _filestorHandler->flush(!downwards);"); for (uint32_t i = 0; i < _disks.size(); ++i) { for (uint32_t j = 0; j < _disks[i].size(); ++j) { - if (_disks[i][j].get() != NULL) { + if (_disks[i][j]) { _disks[i][j]->flush(); LOG(debug, "flushed disk[%d][%d]", i, j); } diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index e5e358cbb60..d47ed28f636 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -91,40 +91,6 @@ public: } }; -class FlushGuard -{ - spi::PersistenceProvider& _spi; - spi::Bucket _bucket; - spi::Context& _context; - bool _hasFlushed; -public: - FlushGuard(spi::PersistenceProvider& spi, - const spi::Bucket& bucket, - spi::Context& context) - : _spi(spi), - _bucket(bucket), - _context(context), - _hasFlushed(false) - {} - ~FlushGuard() - { - if (!_hasFlushed) { - LOG(debug, "Auto-flushing %s", _bucket.toString().c_str()); - spi::Result result =_spi.flush(_bucket, _context); - if (result.hasError()) { - LOG(debug, "Flush %s failed: %s", - _bucket.toString().c_str(), - result.toString().c_str()); - } - } - } - void flush() { - LOG(debug, "Flushing %s", _bucket.toString().c_str()); - _hasFlushed = true; - checkResult(_spi.flush(_bucket, _context), _bucket, "flush"); - } -}; - struct IndirectDocEntryTimestampPredicate { bool operator()(const spi::DocEntry::UP& e1, @@ -607,8 +573,6 @@ MergeHandler::applyDiffLocally( std::vector<spi::DocEntry::UP> entries; populateMetaData(bucket, MAX_TIMESTAMP, entries, context); - FlushGuard flushGuard(_spi, bucket, context); - std::shared_ptr<const document::DocumentTypeRepo> repo(_env._component.getTypeRepo()); assert(repo.get() != nullptr); @@ -702,8 +666,6 @@ MergeHandler::applyDiffLocally( LOG(debug, "Merge(%s): Applied %u entries locally from ApplyBucketDiff.", bucket.toString().c_str(), addedCount); - flushGuard.flush(); - spi::BucketInfoResult infoResult(_spi.getBucketInfo(bucket)); if (infoResult.getErrorCode() != spi::Result::ErrorType::NONE) { LOG(warning, "Failed to get bucket info for %s: %s", diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index 4bcd92293d3..aaf62a85e87 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -29,7 +29,6 @@ PersistenceThread::PersistenceThread(ServiceLayerComponentRegister& compReg, _processAllHandler(_env, provider), _mergeHandler(_spi, _env), _diskMoveHandler(_env, _spi), - _context(documentapi::LoadType::DEFAULT, 0, 0), _bucketOwnershipNotifier(), _flushMonitor(), _closed(false) @@ -86,11 +85,11 @@ bool PersistenceThread::tasConditionExists(const api::TestAndSetCommand & cmd) { } bool PersistenceThread::tasConditionMatches(const api::TestAndSetCommand & cmd, MessageTracker & tracker, - bool missingDocumentImpliesMatch) { + spi::Context & context, bool missingDocumentImpliesMatch) { try { TestAndSetHelper helper(*this, cmd, missingDocumentImpliesMatch); - auto code = helper.retrieveAndMatch(); + auto code = helper.retrieveAndMatch(context); if (code.failed()) { tracker.fail(code.getResult(), code.getMessage()); return false; @@ -105,35 +104,35 @@ bool PersistenceThread::tasConditionMatches(const api::TestAndSetCommand & cmd, } MessageTracker::UP -PersistenceThread::handlePut(api::PutCommand& cmd) +PersistenceThread::handlePut(api::PutCommand& cmd, spi::Context & context) { auto& metrics = _env._metrics.put[cmd.getLoadType()]; auto tracker = std::make_unique<MessageTracker>(metrics, _env._component.getClock()); metrics.request_size.addValue(cmd.getApproxByteSize()); - if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker)) { + if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker, context)) { return tracker; } spi::Result response = _spi.put(getBucket(cmd.getDocumentId(), cmd.getBucket()), - spi::Timestamp(cmd.getTimestamp()), cmd.getDocument(), _context); + spi::Timestamp(cmd.getTimestamp()), cmd.getDocument(), context); checkForError(response, *tracker); return tracker; } MessageTracker::UP -PersistenceThread::handleRemove(api::RemoveCommand& cmd) +PersistenceThread::handleRemove(api::RemoveCommand& cmd, spi::Context & context) { auto& metrics = _env._metrics.remove[cmd.getLoadType()]; auto tracker = std::make_unique<MessageTracker>(metrics,_env._component.getClock()); metrics.request_size.addValue(cmd.getApproxByteSize()); - if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker)) { + if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker, context)) { return tracker; } spi::RemoveResult response = _spi.removeIfFound(getBucket(cmd.getDocumentId(), cmd.getBucket()), - spi::Timestamp(cmd.getTimestamp()), cmd.getDocumentId(), _context); + spi::Timestamp(cmd.getTimestamp()), cmd.getDocumentId(), context); if (checkForError(response, *tracker)) { tracker->setReply(std::make_shared<api::RemoveReply>(cmd, response.wasFound() ? cmd.getTimestamp() : 0)); } @@ -144,18 +143,18 @@ PersistenceThread::handleRemove(api::RemoveCommand& cmd) } MessageTracker::UP -PersistenceThread::handleUpdate(api::UpdateCommand& cmd) +PersistenceThread::handleUpdate(api::UpdateCommand& cmd, spi::Context & context) { auto& metrics = _env._metrics.update[cmd.getLoadType()]; auto tracker = std::make_unique<MessageTracker>(metrics, _env._component.getClock()); metrics.request_size.addValue(cmd.getApproxByteSize()); - if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker, cmd.getUpdate()->getCreateIfNonExistent())) { + if (tasConditionExists(cmd) && !tasConditionMatches(cmd, *tracker, context, cmd.getUpdate()->getCreateIfNonExistent())) { return tracker; } spi::UpdateResult response = _spi.update(getBucket(cmd.getUpdate()->getId(), cmd.getBucket()), - spi::Timestamp(cmd.getTimestamp()), cmd.getUpdate(), _context); + spi::Timestamp(cmd.getTimestamp()), cmd.getUpdate(), context); if (checkForError(response, *tracker)) { auto reply = std::make_shared<api::UpdateReply>(cmd); reply->setOldTimestamp(response.getExistingTimestamp()); @@ -177,7 +176,7 @@ spi::ReadConsistency api_read_consistency_to_spi(api::InternalReadConsistency co } MessageTracker::UP -PersistenceThread::handleGet(api::GetCommand& cmd) +PersistenceThread::handleGet(api::GetCommand& cmd, spi::Context & context) { auto& metrics = _env._metrics.get[cmd.getLoadType()]; auto tracker = std::make_unique<MessageTracker>(metrics,_env._component.getClock()); @@ -185,10 +184,9 @@ PersistenceThread::handleGet(api::GetCommand& cmd) document::FieldSetRepo repo; document::FieldSet::UP fieldSet = repo.parse(*_env._component.getTypeRepo(), cmd.getFieldSet()); - // _context is reset per command, so it's safe to modify it like this. - _context.setReadConsistency(api_read_consistency_to_spi(cmd.internal_read_consistency())); + context.setReadConsistency(api_read_consistency_to_spi(cmd.internal_read_consistency())); spi::GetResult result = - _spi.get(getBucket(cmd.getDocumentId(), cmd.getBucket()), *fieldSet, cmd.getDocumentId(), _context); + _spi.get(getBucket(cmd.getDocumentId(), cmd.getBucket()), *fieldSet, cmd.getDocumentId(), context); if (checkForError(result, *tracker)) { if (!result.hasDocument()) { @@ -227,19 +225,19 @@ PersistenceThread::handleRepairBucket(RepairBucketCommand& cmd) } MessageTracker::UP -PersistenceThread::handleRevert(api::RevertCommand& cmd) +PersistenceThread::handleRevert(api::RevertCommand& cmd, spi::Context & context) { auto tracker = std::make_unique<MessageTracker>(_env._metrics.revert[cmd.getLoadType()],_env._component.getClock()); spi::Bucket b = spi::Bucket(cmd.getBucket(), spi::PartitionId(_env._partition)); const std::vector<api::Timestamp> & tokens = cmd.getRevertTokens(); for (const api::Timestamp & token : tokens) { - spi::Result result = _spi.removeEntry(b, spi::Timestamp(token), _context); + spi::Result result = _spi.removeEntry(b, spi::Timestamp(token), context); } return tracker; } MessageTracker::UP -PersistenceThread::handleCreateBucket(api::CreateBucketCommand& cmd) +PersistenceThread::handleCreateBucket(api::CreateBucketCommand& cmd, spi::Context & context) { auto tracker = std::make_unique<MessageTracker>(_env._metrics.createBuckets,_env._component.getClock()); LOG(debug, "CreateBucket(%s)", cmd.getBucketId().toString().c_str()); @@ -248,7 +246,7 @@ PersistenceThread::handleCreateBucket(api::CreateBucketCommand& cmd) DUMP_LOGGED_BUCKET_OPERATIONS(cmd.getBucketId()); } spi::Bucket spiBucket(cmd.getBucket(), spi::PartitionId(_env._partition)); - _spi.createBucket(spiBucket, _context); + _spi.createBucket(spiBucket, context); if (cmd.getActive()) { _spi.setActiveState(spiBucket, spi::BucketInfo::ACTIVE); } @@ -297,7 +295,7 @@ PersistenceThread::checkProviderBucketInfoMatches(const spi::Bucket& bucket, con } MessageTracker::UP -PersistenceThread::handleDeleteBucket(api::DeleteBucketCommand& cmd) +PersistenceThread::handleDeleteBucket(api::DeleteBucketCommand& cmd, spi::Context & context) { auto tracker = std::make_unique<MessageTracker>(_env._metrics.deleteBuckets,_env._component.getClock()); LOG(debug, "DeletingBucket(%s)", cmd.getBucketId().toString().c_str()); @@ -310,7 +308,7 @@ PersistenceThread::handleDeleteBucket(api::DeleteBucketCommand& cmd) if (!checkProviderBucketInfoMatches(bucket, cmd.getBucketInfo())) { return tracker; } - _spi.deleteBucket(bucket, _context); + _spi.deleteBucket(bucket, context); StorBucketDatabase& db(_env.getBucketDatabase(cmd.getBucket().getBucketSpace())); { StorBucketDatabase::WrappedEntry entry(db.get(cmd.getBucketId(), "FileStorThread::onDeleteBucket")); @@ -335,10 +333,10 @@ PersistenceThread::handleDeleteBucket(api::DeleteBucketCommand& cmd) } MessageTracker::UP -PersistenceThread::handleGetIter(GetIterCommand& cmd) +PersistenceThread::handleGetIter(GetIterCommand& cmd, spi::Context & context) { auto tracker = std::make_unique<MessageTracker>(_env._metrics.visit[cmd.getLoadType()],_env._component.getClock()); - spi::IterateResult result(_spi.iterate(cmd.getIteratorId(), cmd.getMaxByteSize(), _context)); + spi::IterateResult result(_spi.iterate(cmd.getIteratorId(), cmd.getMaxByteSize(), context)); if (checkForError(result, *tracker)) { GetIterReply::SP reply(new GetIterReply(cmd)); reply->getEntries() = result.steal_entries(); @@ -376,16 +374,15 @@ PersistenceThread::handleReadBucketInfo(ReadBucketInfo& cmd) } MessageTracker::UP -PersistenceThread::handleCreateIterator(CreateIteratorCommand& cmd) +PersistenceThread::handleCreateIterator(CreateIteratorCommand& cmd, spi::Context & context) { auto tracker = std::make_unique<MessageTracker>(_env._metrics.createIterator,_env._component.getClock()); document::FieldSetRepo repo; document::FieldSet::UP fieldSet = repo.parse(*_env._component.getTypeRepo(), cmd.getFields()); - // _context is reset per command, so it's safe to modify it like this. - _context.setReadConsistency(cmd.getReadConsistency()); + context.setReadConsistency(cmd.getReadConsistency()); spi::CreateIteratorResult result(_spi.createIterator( spi::Bucket(cmd.getBucket(), spi::PartitionId(_env._partition)), - *fieldSet, cmd.getSelection(), cmd.getIncludedVersions(), _context)); + *fieldSet, cmd.getSelection(), cmd.getIncludedVersions(), context)); if (checkForError(result, *tracker)) { tracker->setReply(std::make_shared<CreateIteratorReply>(cmd, spi::IteratorId(result.getIteratorId()))); } @@ -393,7 +390,7 @@ PersistenceThread::handleCreateIterator(CreateIteratorCommand& cmd) } MessageTracker::UP -PersistenceThread::handleSplitBucket(api::SplitBucketCommand& cmd) +PersistenceThread::handleSplitBucket(api::SplitBucketCommand& cmd, spi::Context & context) { auto tracker = std::make_unique<MessageTracker>(_env._metrics.splitBuckets,_env._component.getClock()); NotificationGuard notifyGuard(*_bucketOwnershipNotifier); @@ -414,7 +411,7 @@ PersistenceThread::handleSplitBucket(api::SplitBucketCommand& cmd) SplitBitDetector::Result targetInfo; if (_env._config.enableMultibitSplitOptimalization) { targetInfo = SplitBitDetector::detectSplit(_spi, spiBucket, cmd.getMaxSplitBits(), - _context, cmd.getMinDocCount(), cmd.getMinByteSize()); + context, cmd.getMinDocCount(), cmd.getMinByteSize()); } if (targetInfo.empty() || !_env._config.enableMultibitSplitOptimalization) { document::BucketId src(cmd.getBucketId()); @@ -454,7 +451,7 @@ PersistenceThread::handleSplitBucket(api::SplitBucketCommand& cmd) } #endif spi::Result result = _spi.split(spiBucket, spi::Bucket(target1, spi::PartitionId(lock1.disk)), - spi::Bucket(target2, spi::PartitionId(lock2.disk)), _context); + spi::Bucket(target2, spi::PartitionId(lock2.disk)), context); if (result.hasError()) { tracker->fail(_env.convertErrorCode(result), result.getErrorMessage()); return tracker; @@ -512,7 +509,7 @@ PersistenceThread::handleSplitBucket(api::SplitBucketCommand& cmd) spi::PartitionId(targets[i].second.diskIndex))); LOG(debug, "Split target %s was empty, but re-creating it since there are remapped operations queued to it", createTarget.toString().c_str()); - _spi.createBucket(createTarget, _context); + _spi.createBucket(createTarget, context); } splitReply.getSplitInfo().emplace_back(targets[i].second.bucket.getBucketId(), targets[i].first->getBucketInfo()); @@ -557,7 +554,7 @@ PersistenceThread::validateJoinCommand(const api::JoinBucketsCommand& cmd, Messa } MessageTracker::UP -PersistenceThread::handleJoinBuckets(api::JoinBucketsCommand& cmd) +PersistenceThread::handleJoinBuckets(api::JoinBucketsCommand& cmd, spi::Context & context) { auto tracker = std::make_unique<MessageTracker>(_env._metrics.joinBuckets,_env._component.getClock()); if (!validateJoinCommand(cmd, *tracker)) { @@ -606,11 +603,7 @@ PersistenceThread::handleJoinBuckets(api::JoinBucketsCommand& cmd) _spi.join(spi::Bucket(firstBucket, spi::PartitionId(lock1.disk)), spi::Bucket(secondBucket, spi::PartitionId(lock2.disk)), spi::Bucket(destBucket, spi::PartitionId(_env._partition)), - _context); - if (!checkForError(result, *tracker)) { - return tracker; - } - result = _spi.flush(spi::Bucket(destBucket, spi::PartitionId(_env._partition)), _context); + context); if (!checkForError(result, *tracker)) { return tracker; } @@ -672,7 +665,7 @@ PersistenceThread::handleSetBucketState(api::SetBucketStateCommand& cmd) } MessageTracker::UP -PersistenceThread::handleInternalBucketJoin(InternalBucketJoinCommand& cmd) +PersistenceThread::handleInternalBucketJoin(InternalBucketJoinCommand& cmd, spi::Context & context) { auto tracker = std::make_unique<MessageTracker>(_env._metrics.internalJoin,_env._component.getClock()); document::Bucket destBucket = cmd.getBucket(); @@ -689,7 +682,7 @@ PersistenceThread::handleInternalBucketJoin(InternalBucketJoinCommand& cmd) _spi.join(spi::Bucket(destBucket, spi::PartitionId(cmd.getDiskOfInstanceToJoin())), spi::Bucket(destBucket, spi::PartitionId(cmd.getDiskOfInstanceToJoin())), spi::Bucket(destBucket, spi::PartitionId(cmd.getDiskOfInstanceToKeep())), - _context); + context); if (checkForError(result, *tracker)) { tracker->setReply(std::make_shared<InternalBucketJoinReply>(cmd, _env.getBucketInfo(cmd.getBucket()))); } @@ -727,46 +720,46 @@ PersistenceThread::handleRecheckBucketInfo(RecheckBucketInfoCommand& cmd) } MessageTracker::UP -PersistenceThread::handleCommandSplitByType(api::StorageCommand& msg) +PersistenceThread::handleCommandSplitByType(api::StorageCommand& msg, spi::Context & context) { switch (msg.getType().getId()) { case api::MessageType::GET_ID: - return handleGet(static_cast<api::GetCommand&>(msg)); + return handleGet(static_cast<api::GetCommand&>(msg), context); case api::MessageType::PUT_ID: - return handlePut(static_cast<api::PutCommand&>(msg)); + return handlePut(static_cast<api::PutCommand&>(msg), context); case api::MessageType::REMOVE_ID: - return handleRemove(static_cast<api::RemoveCommand&>(msg)); + return handleRemove(static_cast<api::RemoveCommand&>(msg), context); case api::MessageType::UPDATE_ID: - return handleUpdate(static_cast<api::UpdateCommand&>(msg)); + return handleUpdate(static_cast<api::UpdateCommand&>(msg), context); case api::MessageType::REVERT_ID: - return handleRevert(static_cast<api::RevertCommand&>(msg)); + return handleRevert(static_cast<api::RevertCommand&>(msg), context); case api::MessageType::CREATEBUCKET_ID: - return handleCreateBucket(static_cast<api::CreateBucketCommand&>(msg)); + return handleCreateBucket(static_cast<api::CreateBucketCommand&>(msg), context); case api::MessageType::DELETEBUCKET_ID: - return handleDeleteBucket(static_cast<api::DeleteBucketCommand&>(msg)); + return handleDeleteBucket(static_cast<api::DeleteBucketCommand&>(msg), context); case api::MessageType::JOINBUCKETS_ID: - return handleJoinBuckets(static_cast<api::JoinBucketsCommand&>(msg)); + return handleJoinBuckets(static_cast<api::JoinBucketsCommand&>(msg), context); case api::MessageType::SPLITBUCKET_ID: - return handleSplitBucket(static_cast<api::SplitBucketCommand&>(msg)); + return handleSplitBucket(static_cast<api::SplitBucketCommand&>(msg), context); // Depends on iterators case api::MessageType::STATBUCKET_ID: - return _processAllHandler.handleStatBucket(static_cast<api::StatBucketCommand&>(msg), _context); + return _processAllHandler.handleStatBucket(static_cast<api::StatBucketCommand&>(msg), context); case api::MessageType::REMOVELOCATION_ID: - return _processAllHandler.handleRemoveLocation(static_cast<api::RemoveLocationCommand&>(msg), _context); + return _processAllHandler.handleRemoveLocation(static_cast<api::RemoveLocationCommand&>(msg), context); case api::MessageType::MERGEBUCKET_ID: - return _mergeHandler.handleMergeBucket(static_cast<api::MergeBucketCommand&>(msg), _context); + return _mergeHandler.handleMergeBucket(static_cast<api::MergeBucketCommand&>(msg), context); case api::MessageType::GETBUCKETDIFF_ID: - return _mergeHandler.handleGetBucketDiff(static_cast<api::GetBucketDiffCommand&>(msg), _context); + return _mergeHandler.handleGetBucketDiff(static_cast<api::GetBucketDiffCommand&>(msg), context); case api::MessageType::APPLYBUCKETDIFF_ID: - return _mergeHandler.handleApplyBucketDiff(static_cast<api::ApplyBucketDiffCommand&>(msg), _context); + return _mergeHandler.handleApplyBucketDiff(static_cast<api::ApplyBucketDiffCommand&>(msg), context); case api::MessageType::SETBUCKETSTATE_ID: return handleSetBucketState(static_cast<api::SetBucketStateCommand&>(msg)); case api::MessageType::INTERNAL_ID: switch(static_cast<api::InternalCommand&>(msg).getType()) { case GetIterCommand::ID: - return handleGetIter(static_cast<GetIterCommand&>(msg)); + return handleGetIter(static_cast<GetIterCommand&>(msg), context); case CreateIteratorCommand::ID: - return handleCreateIterator(static_cast<CreateIteratorCommand&>(msg)); + return handleCreateIterator(static_cast<CreateIteratorCommand&>(msg), context); case ReadBucketList::ID: return handleReadBucketList(static_cast<ReadBucketList&>(msg)); case ReadBucketInfo::ID: @@ -774,9 +767,9 @@ PersistenceThread::handleCommandSplitByType(api::StorageCommand& msg) case RepairBucketCommand::ID: return handleRepairBucket(static_cast<RepairBucketCommand&>(msg)); case BucketDiskMoveCommand::ID: - return _diskMoveHandler.handleBucketDiskMove(static_cast<BucketDiskMoveCommand&>(msg), _context); + return _diskMoveHandler.handleBucketDiskMove(static_cast<BucketDiskMoveCommand&>(msg), context); case InternalBucketJoinCommand::ID: - return handleInternalBucketJoin(static_cast<InternalBucketJoinCommand&>(msg)); + return handleInternalBucketJoin(static_cast<InternalBucketJoinCommand&>(msg), context); case RecheckBucketInfoCommand::ID: return handleRecheckBucketInfo(static_cast<RecheckBucketInfoCommand&>(msg)); default: @@ -792,13 +785,13 @@ PersistenceThread::handleCommandSplitByType(api::StorageCommand& msg) MessageTracker::UP PersistenceThread::handleCommand(api::StorageCommand& msg) { - _context = spi::Context(msg.getLoadType(), msg.getPriority(), msg.getTrace().getLevel()); - MessageTracker::UP mtracker(handleCommandSplitByType(msg)); - if (mtracker && ! _context.getTrace().getRoot().isEmpty()) { + spi::Context context(msg.getLoadType(), msg.getPriority(), msg.getTrace().getLevel()); + MessageTracker::UP mtracker(handleCommandSplitByType(msg, context)); + if (mtracker && ! context.getTrace().getRoot().isEmpty()) { if (mtracker->getReply()) { - mtracker->getReply()->getTrace().getRoot().addChild(_context.getTrace().getRoot()); + mtracker->getReply()->getTrace().getRoot().addChild(context.getTrace().getRoot()); } else { - msg.getTrace().getRoot().addChild(_context.getTrace().getRoot()); + msg.getTrace().getRoot().addChild(context.getTrace().getRoot()); } } return mtracker; @@ -898,111 +891,24 @@ bool hasBucketInfo(const api::StorageMessage& msg) } void -PersistenceThread::flushAllReplies( - const document::Bucket& bucket, - std::vector<std::unique_ptr<MessageTracker> >& replies) -{ - if (replies.empty()) { - return; - } - - try { - if (replies.size() > 1) { - _env._metrics.batchingSize.addValue(replies.size()); - } -#ifdef ENABLE_BUCKET_OPERATION_LOGGING - { - size_t nputs = 0, nremoves = 0, nother = 0; - for (size_t i = 0; i < replies.size(); ++i) { - if (dynamic_cast<api::PutReply*>(replies[i]->getReply().get())) - { - ++nputs; - } else if (dynamic_cast<api::RemoveReply*>( - replies[i]->getReply().get())) - { - ++nremoves; - } else { - ++nother; - } - } - LOG_BUCKET_OPERATION( - bucket.getBucketId(), - vespalib::make_string( - "flushing %zu operations (%zu puts, %zu removes, " - "%zu other)", - replies.size(), nputs, nremoves, nother)); - } -#endif - spi::Bucket b(bucket, spi::PartitionId(_env._partition)); - spi::Result result = _spi.flush(b, _context); - uint32_t errorCode = _env.convertErrorCode(result); - if (errorCode != 0) { - for (uint32_t i = 0; i < replies.size(); ++i) { - replies[i]->getReply()->setResult(api::ReturnCode((api::ReturnCode::Result)errorCode, result.getErrorMessage())); - } - } - } catch (std::exception& e) { - for (uint32_t i = 0; i < replies.size(); ++i) { - replies[i]->getReply()->setResult(api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE, e.what())); - } - } - - for (uint32_t i = 0; i < replies.size(); ++i) { - LOG(spam, "Sending reply up (batched): %s %" PRIu64, - replies[i]->getReply()->toString().c_str(), replies[i]->getReply()->getMsgId()); - _env._fileStorHandler.sendReply(replies[i]->getReply()); - } - - replies.clear(); -} - -void PersistenceThread::processMessages(FileStorHandler::LockedMessage & lock) -{ +PersistenceThread::processLockedMessage(FileStorHandler::LockedMessage & lock) { std::vector<MessageTracker::UP> trackers; document::Bucket bucket = lock.first->getBucket(); - while (lock.second) { - LOG(debug, "Inside while loop %d, nodeIndex %d, ptr=%p", _env._partition, _env._nodeIndex, lock.second.get()); - std::shared_ptr<api::StorageMessage> msg(lock.second); - bool batchable = isBatchable(*msg); - - // If the next operation wasn't batchable, we should flush - // everything that came before. - if (!batchable) { - flushAllReplies(bucket, trackers); - } + LOG(debug, "Partition %d, nodeIndex %d, ptr=%p", _env._partition, _env._nodeIndex, lock.second.get()); + api::StorageMessage & msg(*lock.second); - std::unique_ptr<MessageTracker> tracker = processMessage(*msg); - if (!tracker || !tracker->getReply()) { - // Was a reply - break; - } - - if (hasBucketInfo(*msg)) { + std::unique_ptr<MessageTracker> tracker = processMessage(msg); + if (tracker && tracker->getReply()) { + if (hasBucketInfo(msg)) { if (tracker->getReply()->getResult().success()) { _env.setBucketInfo(*tracker, bucket); } } - if (batchable) { - LOG(spam, "Adding reply %s to batch for bucket %s", - tracker->getReply()->toString().c_str(), bucket.getBucketId().toString().c_str()); - - trackers.push_back(std::move(tracker)); - - if (trackers.back()->getReply()->getResult().success()) { - _env._fileStorHandler.getNextMessage(_env._partition, _stripeId, lock); - } else { - break; - } - } else { - LOG(spam, "Sending reply up: %s %" PRIu64, - tracker->getReply()->toString().c_str(), tracker->getReply()->getMsgId()); - _env._fileStorHandler.sendReply(tracker->getReply()); - break; - } + LOG(spam, "Sending reply up: %s %" PRIu64, + tracker->getReply()->toString().c_str(), tracker->getReply()->getMsgId()); + _env._fileStorHandler.sendReply(tracker->getReply()); } - - flushAllReplies(bucket, trackers); } void @@ -1016,7 +922,7 @@ PersistenceThread::run(framework::ThreadHandle& thread) FileStorHandler::LockedMessage lock(_env._fileStorHandler.getNextMessage(_env._partition, _stripeId)); if (lock.first) { - processMessages(lock); + processLockedMessage(lock); } vespalib::MonitorGuard flushMonitorGuard(_flushMonitor); diff --git a/storage/src/vespa/storage/persistence/persistencethread.h b/storage/src/vespa/storage/persistence/persistencethread.h index ed27a759e8b..56414835b7b 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.h +++ b/storage/src/vespa/storage/persistence/persistencethread.h @@ -22,27 +22,27 @@ public: PersistenceThread(ServiceLayerComponentRegister&, const config::ConfigUri & configUri, spi::PersistenceProvider& provider, FileStorHandler& filestorHandler, FileStorThreadMetrics& metrics, uint16_t deviceIndex); - ~PersistenceThread(); + ~PersistenceThread() override; /** Waits for current operation to be finished. */ void flush() override; framework::Thread& getThread() override { return *_thread; } - MessageTracker::UP handlePut(api::PutCommand& cmd); - MessageTracker::UP handleRemove(api::RemoveCommand& cmd); - MessageTracker::UP handleUpdate(api::UpdateCommand& cmd); - MessageTracker::UP handleGet(api::GetCommand& cmd); - MessageTracker::UP handleRevert(api::RevertCommand& cmd); - MessageTracker::UP handleCreateBucket(api::CreateBucketCommand& cmd); - MessageTracker::UP handleDeleteBucket(api::DeleteBucketCommand& cmd); - MessageTracker::UP handleCreateIterator(CreateIteratorCommand& cmd); - MessageTracker::UP handleGetIter(GetIterCommand& cmd); + MessageTracker::UP handlePut(api::PutCommand& cmd, spi::Context & context); + MessageTracker::UP handleRemove(api::RemoveCommand& cmd, spi::Context & context); + MessageTracker::UP handleUpdate(api::UpdateCommand& cmd, spi::Context & context); + MessageTracker::UP handleGet(api::GetCommand& cmd, spi::Context & context); + MessageTracker::UP handleRevert(api::RevertCommand& cmd, spi::Context & context); + MessageTracker::UP handleCreateBucket(api::CreateBucketCommand& cmd, spi::Context & context); + MessageTracker::UP handleDeleteBucket(api::DeleteBucketCommand& cmd, spi::Context & context); + MessageTracker::UP handleCreateIterator(CreateIteratorCommand& cmd, spi::Context & context); + MessageTracker::UP handleGetIter(GetIterCommand& cmd, spi::Context & context); MessageTracker::UP handleReadBucketList(ReadBucketList& cmd); MessageTracker::UP handleReadBucketInfo(ReadBucketInfo& cmd); - MessageTracker::UP handleJoinBuckets(api::JoinBucketsCommand& cmd); + MessageTracker::UP handleJoinBuckets(api::JoinBucketsCommand& cmd, spi::Context & context); MessageTracker::UP handleSetBucketState(api::SetBucketStateCommand& cmd); - MessageTracker::UP handleInternalBucketJoin(InternalBucketJoinCommand& cmd); - MessageTracker::UP handleSplitBucket(api::SplitBucketCommand& cmd); + MessageTracker::UP handleInternalBucketJoin(InternalBucketJoinCommand& cmd, spi::Context & context); + MessageTracker::UP handleSplitBucket(api::SplitBucketCommand& cmd, spi::Context & context); MessageTracker::UP handleRepairBucket(RepairBucketCommand& cmd); MessageTracker::UP handleRecheckBucketInfo(RecheckBucketInfoCommand& cmd); @@ -56,7 +56,6 @@ private: DiskMoveOperationHandler _diskMoveHandler; ServiceLayerComponent::UP _component; framework::Thread::UP _thread; - spi::Context _context; std::unique_ptr<BucketOwnershipNotifier> _bucketOwnershipNotifier; vespalib::Monitor _flushMonitor; bool _closed; @@ -72,23 +71,21 @@ private: // Message handling functions MessageTracker::UP handleCommand(api::StorageCommand&); - MessageTracker::UP handleCommandSplitByType(api::StorageCommand&); + MessageTracker::UP handleCommandSplitByType(api::StorageCommand&, spi::Context & context); void handleReply(api::StorageReply&); MessageTracker::UP processMessage(api::StorageMessage& msg); - void processMessages(FileStorHandler::LockedMessage & lock); + void processLockedMessage(FileStorHandler::LockedMessage & lock); // Thread main loop void run(framework::ThreadHandle&) override; bool checkForError(const spi::Result& response, MessageTracker& tracker); spi::Bucket getBucket(const DocumentId& id, const document::Bucket &bucket) const; - void flushAllReplies(const document::Bucket& bucket, std::vector<MessageTracker::UP>& trackers); - friend class TestAndSetHelper; bool tasConditionExists(const api::TestAndSetCommand & cmd); bool tasConditionMatches(const api::TestAndSetCommand & cmd, MessageTracker & tracker, - bool missingDocumentImpliesMatch = false); + spi::Context & context, bool missingDocumentImpliesMatch = false); }; } // storage diff --git a/storage/src/vespa/storage/persistence/processallhandler.cpp b/storage/src/vespa/storage/persistence/processallhandler.cpp index 5b94a3da027..f37c6723933 100644 --- a/storage/src/vespa/storage/persistence/processallhandler.cpp +++ b/storage/src/vespa/storage/persistence/processallhandler.cpp @@ -83,9 +83,9 @@ MessageTracker::UP ProcessAllHandler::handleRemoveLocation(api::RemoveLocationCommand& cmd, spi::Context& context) { - MessageTracker::UP tracker(new MessageTracker( + auto tracker = std::make_unique<MessageTracker>( _env._metrics.removeLocation[cmd.getLoadType()], - _env._component.getClock())); + _env._component.getClock()); LOG(debug, "RemoveLocation(%s): using selection '%s'", cmd.getBucketId().toString().c_str(), @@ -99,13 +99,8 @@ ProcessAllHandler::handleRemoveLocation(api::RemoveLocationCommand& cmd, processor, spi::NEWEST_DOCUMENT_ONLY, context); - spi::Result result = _spi.flush(bucket, context); - uint32_t code = _env.convertErrorCode(result); - if (code == 0) { - tracker->setReply(std::make_shared<api::RemoveLocationReply>(cmd, processor._n_removed)); - } else { - tracker->fail(code, result.getErrorMessage()); - } + + tracker->setReply(std::make_shared<api::RemoveLocationReply>(cmd, processor._n_removed)); return tracker; } @@ -114,9 +109,9 @@ MessageTracker::UP ProcessAllHandler::handleStatBucket(api::StatBucketCommand& cmd, spi::Context& context) { - MessageTracker::UP tracker(new MessageTracker( + auto tracker = std::make_unique<MessageTracker>( _env._metrics.statBucket[cmd.getLoadType()], - _env._component.getClock())); + _env._component.getClock()); std::ostringstream ost; ost << "Persistence bucket " << cmd.getBucketId() diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp index fe947bff4d5..76c420e76e6 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.cpp @@ -118,12 +118,6 @@ ProviderErrorWrapper::get(const spi::Bucket& bucket, return checkResult(_impl.get(bucket, fieldSet, docId, context)); } -spi::Result -ProviderErrorWrapper::flush(const spi::Bucket& bucket, spi::Context& context) -{ - return checkResult(_impl.flush(bucket, context)); -} - spi::CreateIteratorResult ProviderErrorWrapper::createIterator(const spi::Bucket& bucket, const document::FieldSet& fieldSet, diff --git a/storage/src/vespa/storage/persistence/provider_error_wrapper.h b/storage/src/vespa/storage/persistence/provider_error_wrapper.h index 3b5ace90d13..292eb004223 100644 --- a/storage/src/vespa/storage/persistence/provider_error_wrapper.h +++ b/storage/src/vespa/storage/persistence/provider_error_wrapper.h @@ -52,7 +52,6 @@ public: spi::RemoveResult removeIfFound(const spi::Bucket&, spi::Timestamp, const document::DocumentId&, spi::Context&) override; spi::UpdateResult update(const spi::Bucket&, spi::Timestamp, const spi::DocumentUpdateSP&, spi::Context&) override; spi::GetResult get(const spi::Bucket&, const document::FieldSet&, const document::DocumentId&, spi::Context&) const override; - spi::Result flush(const spi::Bucket&, spi::Context&) override; spi::CreateIteratorResult createIterator(const spi::Bucket&, const document::FieldSet&, const spi::Selection&, spi::IncludedVersions versions, spi::Context&) override; spi::IterateResult iterate(spi::IteratorId, uint64_t maxByteSize, spi::Context&) const override; diff --git a/storage/src/vespa/storage/persistence/testandsethelper.cpp b/storage/src/vespa/storage/persistence/testandsethelper.cpp index 511d44ad331..e1909252c8f 100644 --- a/storage/src/vespa/storage/persistence/testandsethelper.cpp +++ b/storage/src/vespa/storage/persistence/testandsethelper.cpp @@ -31,12 +31,12 @@ void TestAndSetHelper::parseDocumentSelection() { } } -spi::GetResult TestAndSetHelper::retrieveDocument(const document::FieldSet & fieldSet) { +spi::GetResult TestAndSetHelper::retrieveDocument(const document::FieldSet & fieldSet, spi::Context & context) { return _thread._spi.get( _thread.getBucket(_docId, _cmd.getBucket()), fieldSet, _cmd.getDocumentId(), - _thread._context); + context); } TestAndSetHelper::TestAndSetHelper(PersistenceThread & thread, const api::TestAndSetCommand & cmd, @@ -51,16 +51,16 @@ TestAndSetHelper::TestAndSetHelper(PersistenceThread & thread, const api::TestAn parseDocumentSelection(); } -TestAndSetHelper::~TestAndSetHelper() { -} +TestAndSetHelper::~TestAndSetHelper() = default; -api::ReturnCode TestAndSetHelper::retrieveAndMatch() { +api::ReturnCode +TestAndSetHelper::retrieveAndMatch(spi::Context & context) { // Walk document selection tree to build a minimal field set FieldVisitor fieldVisitor(*_docTypePtr); _docSelectionUp->visit(fieldVisitor); // Retrieve document - auto result = retrieveDocument(fieldVisitor.getFieldSet()); + auto result = retrieveDocument(fieldVisitor.getFieldSet(), context); // If document exists, match it with selection if (result.hasDocument()) { diff --git a/storage/src/vespa/storage/persistence/testandsethelper.h b/storage/src/vespa/storage/persistence/testandsethelper.h index 21c111c712f..b5fa29d0106 100644 --- a/storage/src/vespa/storage/persistence/testandsethelper.h +++ b/storage/src/vespa/storage/persistence/testandsethelper.h @@ -30,13 +30,13 @@ class TestAndSetHelper { void getDocumentType(); void parseDocumentSelection(); - spi::GetResult retrieveDocument(const document::FieldSet & fieldSet); + spi::GetResult retrieveDocument(const document::FieldSet & fieldSet, spi::Context & context); public: TestAndSetHelper(PersistenceThread & thread, const api::TestAndSetCommand & cmd, bool missingDocumentImpliesMatch = false); ~TestAndSetHelper(); - api::ReturnCode retrieveAndMatch(); + api::ReturnCode retrieveAndMatch(spi::Context & context); }; } // storage diff --git a/storageframework/src/tests/thread/tickingthreadtest.cpp b/storageframework/src/tests/thread/tickingthreadtest.cpp index f69f77e8a07..8e3d7b29cd6 100644 --- a/storageframework/src/tests/thread/tickingthreadtest.cpp +++ b/storageframework/src/tests/thread/tickingthreadtest.cpp @@ -273,7 +273,7 @@ struct BroadcastApp : public TickingThread { ThreadWaitInfo doNonCriticalTick(ThreadIndex) override { if (!_active.empty()) { for (uint32_t i=0; i<_active.size(); ++i) { - printTaskInfo(_queue[i], "processing"); + printTaskInfo(_active[i], "processing"); _processed.push_back(_active[i]); } _active.clear(); diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index c43735e7e41..f0b48faebef 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -932,9 +932,9 @@ TEST(DistributionTest, test_hierarchical_no_redistribution) std::vector<uint16_t> nodes; std::vector< std::vector<uint16_t> > distr(4); - int numBuckets = 1000; + size_t numBuckets = 1000; - for (int i = 0; i < numBuckets; i++) { + for (size_t i = 0; i < numBuckets; i++) { nodes = distribution.getIdealStorageNodes( state, document::BucketId(16, i), "u"); for (uint16_t j=0; j<nodes.size(); ++j) { @@ -943,23 +943,22 @@ TEST(DistributionTest, test_hierarchical_no_redistribution) nodes.clear(); } - std::vector<uint16_t>::iterator it; - std::vector<uint16_t> v(1000); + std::vector<uint16_t> v; - it=set_intersection (distr[0].begin(), distr[0].end(), distr[1].begin(), distr[1].end(), v.begin()); - EXPECT_EQ(0, int(it-v.begin())); + set_intersection (distr[0].begin(), distr[0].end(), distr[1].begin(), distr[1].end(), back_inserter(v)); + EXPECT_EQ(0, v.size()); v.clear(); - it=set_intersection (distr[2].begin(), distr[2].end(), distr[3].begin(), distr[3].end(), v.begin()); - EXPECT_EQ(0, int(it-v.begin())); + set_intersection (distr[2].begin(), distr[2].end(), distr[3].begin(), distr[3].end(), back_inserter(v)); + EXPECT_EQ(0, v.size()); v.clear(); - it=set_union (distr[0].begin(), distr[0].end(), distr[1].begin(), distr[1].end(), v.begin()); - EXPECT_EQ(numBuckets, int(it-v.begin())); + set_union (distr[0].begin(), distr[0].end(), distr[1].begin(), distr[1].end(), back_inserter(v)); + EXPECT_EQ(numBuckets, v.size()); v.clear(); - it=set_union (distr[2].begin(), distr[2].end(), distr[3].begin(), distr[3].end(), v.begin()); - EXPECT_EQ(numBuckets, int(it-v.begin())); + set_union (distr[2].begin(), distr[2].end(), distr[3].begin(), distr[3].end(), back_inserter(v)); + EXPECT_EQ(numBuckets, v.size()); v.clear(); state.setNodeState(Node(NodeType::STORAGE, 0), @@ -967,7 +966,7 @@ TEST(DistributionTest, test_hierarchical_no_redistribution) std::vector< std::vector<uint16_t> > distr2(4); - for (int i = 0; i < numBuckets; i++) { + for (size_t i = 0; i < numBuckets; i++) { nodes = distribution.getIdealStorageNodes( state, document::BucketId(16, i), "u"); for (uint16_t j=0; j<nodes.size(); ++j) { @@ -980,16 +979,16 @@ TEST(DistributionTest, test_hierarchical_no_redistribution) ASSERT_EQ((size_t)0, distr2[0].size()); v.clear(); - it=set_difference (distr[1].begin(), distr[1].end(), distr2[1].begin(), distr2[1].end(), v.begin()); - EXPECT_EQ(0, int(it-v.begin())); + set_difference (distr[1].begin(), distr[1].end(), distr2[1].begin(), distr2[1].end(), back_inserter(v)); + EXPECT_EQ(0, v.size()); v.clear(); - it=set_difference (distr[2].begin(), distr[2].end(), distr2[2].begin(), distr2[2].end(), v.begin()); - EXPECT_EQ(0, int(it-v.begin())); + set_difference (distr[2].begin(), distr[2].end(), distr2[2].begin(), distr2[2].end(), back_inserter(v)); + EXPECT_EQ(0, v.size()); v.clear(); - it=set_difference (distr[3].begin(), distr[3].end(), distr2[3].begin(), distr2[3].end(), v.begin()); - EXPECT_EQ(0, int(it-v.begin())); + set_difference (distr[3].begin(), distr[3].end(), distr2[3].begin(), distr2[3].end(), back_inserter(v)); + EXPECT_EQ(0, v.size()); v.clear(); state = ClusterState( diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index 144ffaea5b4..71a4c1a9954 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -10,7 +10,7 @@ import com.yahoo.container.core.identity.IdentityConfig; import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.Pkcs10Csr; @@ -305,7 +305,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen ? athenzCredentialsService.registerInstance() : athenzCredentialsService.updateCredentials(credentials.getIdentityDocument(), identitySslContext)); } catch (Throwable t) { - log.log(LogLevel.WARNING, "Failed to update credentials: " + t.getMessage(), t); + log.log(Level.WARNING, "Failed to update credentials: " + t.getMessage(), t); } } @@ -315,7 +315,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen Duration remainingLifetime = Duration.between(clock.instant(), expirationTime); metric.set(CERTIFICATE_EXPIRY_METRIC_NAME, remainingLifetime.getSeconds(), null); } catch (Throwable t) { - log.log(LogLevel.WARNING, "Failed to update metrics: " + t.getMessage(), t); + log.log(Level.WARNING, "Failed to update metrics: " + t.getMessage(), t); } } } diff --git a/vespa-maven-plugin/pom.xml b/vespa-maven-plugin/pom.xml index de3e2f44a67..9f1d6f5ff6b 100644 --- a/vespa-maven-plugin/pom.xml +++ b/vespa-maven-plugin/pom.xml @@ -41,11 +41,6 @@ </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> <artifactId>config-application-package</artifactId> <version>${project.version}</version> </dependency> diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java index bf95b2ef6c5..7a94bbc1e47 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunner.java @@ -29,9 +29,9 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; -import static com.yahoo.log.LogLevel.ERROR; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.logging.Level.INFO; +import static java.util.logging.Level.SEVERE; /** * @author valerijf @@ -167,7 +167,7 @@ public class TestRunner { success = mavenProcess.waitFor() == 0; } catch (Exception exception) { - LogRecord record = new LogRecord(ERROR, "Failed to execute maven command: " + String.join(" ", builder.command())); + LogRecord record = new LogRecord(SEVERE, "Failed to execute maven command: " + String.join(" ", builder.command())); record.setThrown(exception); logger.log(record); log.put(record.getSequenceNumber(), record); diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java index 90f7a76b356..e92dbcede5a 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/TestRunnerHandler.java @@ -9,7 +9,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.container.logging.AccessLog; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.slime.Cursor; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.Slime; @@ -119,9 +119,9 @@ public class TestRunnerHandler extends LoggingRequestHandler { public static String typeOf(Level level) { return level.getName().equals("html") ? "html" - : level.intValue() < LogLevel.INFO.intValue() ? "debug" - : level.intValue() < LogLevel.WARNING.intValue() ? "info" - : level.intValue() < LogLevel.ERROR.intValue() ? "warning" + : level.intValue() < Level.INFO.intValue() ? "debug" + : level.intValue() < Level.WARNING.intValue() ? "info" + : level.intValue() < Level.SEVERE.intValue() ? "warning" : "error"; } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java index 3b463e1af92..ccd2d80efb2 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/RestApi.java @@ -27,7 +27,7 @@ import com.yahoo.document.select.parser.ParseException; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.text.Text; import com.yahoo.vespa.config.content.LoadTypeConfig; @@ -239,7 +239,7 @@ public class RestApi extends LoggingRequestHandler { RestUri.apiErrorCodes.PARSER_ERROR); } catch (RuntimeException systemException) { - log.log(LogLevel.WARNING, "Internal runtime exception during Document V1 request handling", systemException); + log.log(Level.WARNING, "Internal runtime exception during Document V1 request handling", systemException); return Response.createErrorResponse(500, Exceptions.toMessageString(systemException), restUri, RestUri.apiErrorCodes.UNSPECIFIED); diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java index e6bc2211bea..db1c2471752 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/ClientFeederV3.java @@ -8,7 +8,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Result; @@ -144,7 +144,7 @@ class ClientFeederV3 { } catch (InterruptedException e) { // NOP, just terminate } catch (Throwable e) { - log.log(LogLevel.WARNING, "Unhandled exception while feeding: " + Exceptions.toMessageString(e), e); + log.log(Level.WARNING, "Unhandled exception while feeding: " + Exceptions.toMessageString(e), e); } finally { replies.add(createOperationStatus("-", "-", ErrorCode.END_OF_FEED, false, null)); } @@ -169,8 +169,8 @@ class ClientFeederV3 { try { operationId = streamReaderV3.getNextOperationId(requestInputStream); } catch (IOException ioe) { - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, Exceptions.toMessageString(ioe), ioe); + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, Exceptions.toMessageString(ioe), ioe); } return Optional.empty(); } @@ -182,8 +182,8 @@ class ClientFeederV3 { try { message = getNextMessage(operationId.get(), requestInputStream, settings); } catch (Exception e) { - if (log.isLoggable(LogLevel.WARNING)) { - log.log(LogLevel.WARNING, Exceptions.toMessageString(e)); + if (log.isLoggable(Level.WARNING)) { + log.log(Level.WARNING, Exceptions.toMessageString(e)); } metric.add(MetricNames.PARSE_ERROR, 1, null); @@ -246,7 +246,7 @@ class ClientFeederV3 { if (result.isAccepted()) { outstandingOperations.incrementAndGet(); updateOpsPerSec(); - log(LogLevel.DEBUG, "Sent message successfully, document id: ", msg.get().getOperationId()); + log(Level.FINE, "Sent message successfully, document id: ", msg.get().getOperationId()); } else if (!result.getError().isFatal()) { repliesFromOldMessages.add(createOperationStatus(msg.get().getOperationId(), result.getError().getMessage(), @@ -297,7 +297,7 @@ class ClientFeederV3 { return null; } metric.add(MetricNames.NUM_OPERATIONS, 1, null /*metricContext*/); - log(LogLevel.DEBUG, "Successfully deserialized document id: ", message.getOperationId()); + log(Level.FINE, "Successfully deserialized document id: ", message.getOperationId()); return message; } @@ -325,7 +325,7 @@ class ClientFeederV3 { } } - protected final void log(LogLevel level, Object... msgParts) { + protected final void log(Level level, Object... msgParts) { StringBuilder s; if (!log.isLoggable(level)) { diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java index bcd78c790fe..05a64d633a4 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedHandlerV3.java @@ -12,7 +12,7 @@ import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.metrics.DocumentApiMetrics; import com.yahoo.jdisc.Metric; import com.yahoo.jdisc.ReferencedResource; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.shared.SharedSourceSession; @@ -116,11 +116,11 @@ public class FeedHandlerV3 extends LoggingRequestHandler { return clientFeederV3.handleRequest(request); } catch (UnknownClientException uce) { String msg = Exceptions.toMessageString(uce); - log.log(LogLevel.WARNING, msg); + log.log(Level.WARNING, msg); return new ErrorHttpResponse(com.yahoo.jdisc.http.HttpResponse.Status.BAD_REQUEST, msg); } catch (Exception e) { String msg = "Could not initialize document parsing: " + Exceptions.toMessageString(e); - log.log(LogLevel.WARNING, msg); + log.log(Level.WARNING, msg); return new ErrorHttpResponse(com.yahoo.jdisc.http.HttpResponse.Status.INTERNAL_SERVER_ERROR, msg); } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java index 752f81072e5..5c0c4ffaf3d 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java @@ -9,7 +9,7 @@ import com.yahoo.documentapi.metrics.DocumentApiMetrics; import com.yahoo.documentapi.metrics.DocumentOperationStatus; import com.yahoo.documentapi.metrics.DocumentOperationType; import com.yahoo.jdisc.Metric; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Reply; import com.yahoo.messagebus.ReplyHandler; import com.yahoo.messagebus.Trace; @@ -62,7 +62,7 @@ public class FeedReplyReader implements ReplyHandler { context.feedReplies.put(new OperationStatus(message, context.docId, status, isConditionNotMet, traceMessage)); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, + log.log(Level.WARNING, "Interrupted while enqueueing result from putting document with id: " + context.docId); Thread.currentThread().interrupt(); } diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/DocprocMessageProcessor.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/DocprocMessageProcessor.java deleted file mode 100755 index c6974cff5c1..00000000000 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/DocprocMessageProcessor.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.feedapi; - -import com.yahoo.component.provider.ComponentRegistry; -import com.yahoo.docproc.CallStack; -import com.yahoo.docproc.DocprocService; -import com.yahoo.docproc.DocumentProcessor; -import com.yahoo.docproc.Processing; -import com.yahoo.document.DocumentOperation; -import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; -import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; -import com.yahoo.messagebus.Message; -import com.yahoo.messagebus.routing.Route; - -import java.util.ArrayList; -import java.util.List; - -public class DocprocMessageProcessor implements MessageProcessor { - private final DocprocService docproc; - private final ComponentRegistry<DocprocService> docprocServiceRegistry; - - public DocprocMessageProcessor(DocprocService docproc, ComponentRegistry<DocprocService> docprocServiceRegistry) { - this.docproc = docproc; - this.docprocServiceRegistry = docprocServiceRegistry; - } - - @Override - public void process(Message m) { - try { - List<DocumentOperation> documentBases = new ArrayList<>(1); - - if (m.getType() == DocumentProtocol.MESSAGE_PUTDOCUMENT) { - documentBases.add(((PutDocumentMessage) m).getDocumentPut()); - } else if (m.getType() == DocumentProtocol.MESSAGE_UPDATEDOCUMENT) { - documentBases.add(((UpdateDocumentMessage) m).getDocumentUpdate()); - } else if (m.getType() == DocumentProtocol.MESSAGE_REMOVEDOCUMENT) { - documentBases.add(((RemoveDocumentMessage) m).getDocumentRemove()); - } - - if (docproc != null) { - processDocumentOperations(documentBases, m); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private void processDocumentOperations(List<DocumentOperation> documentOperations, Message m) throws Exception { - Processing processing = Processing.createProcessingFromDocumentOperations(docproc.getName(), documentOperations, new CallStack(docproc.getCallStack())); - processing.setServiceName(docproc.getName()); - processing.setDocprocServiceRegistry(docprocServiceRegistry); - processing.setVariable("route", m.getRoute()); - processing.setVariable("timeout", m.getTimeRemaining()); - - DocumentProcessor.Progress progress = docproc.getExecutor().process(processing); - while (DocumentProcessor.Progress.LATER.equals(progress)) { - Thread.sleep(50); - progress = docproc.getExecutor().process(processing); - } - - if (progress == DocumentProcessor.Progress.FAILED - || progress == DocumentProcessor.Progress.PERMANENT_FAILURE) { - throw new RuntimeException("Processing of " + documentOperations + " failed: " + progress + "."); - } - - m.setRoute((Route) processing.getVariable("route")); - m.setTimeRemaining((Long) processing.getVariable("timeout")); - } - -} diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java index 581a51ffc02..0a070c9fe35 100644 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/MessagePropertyProcessor.java @@ -10,7 +10,7 @@ import com.yahoo.documentapi.messagebus.loadtypes.LoadType; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; import com.yahoo.documentapi.messagebus.protocol.DocumentMessage; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.routing.Route; import com.yahoo.vespaclient.config.FeederConfig; @@ -152,8 +152,8 @@ public class MessagePropertyProcessor implements ConfigSubscriber.SingleSubscrib defaultAbortOnDocumentError = feederOptions.abortOnDocumentError(); defaultAbortOnSendError = feederOptions.abortOnSendError(); - if (log.isLoggable(LogLevel.DEBUG)) { - log.log(LogLevel.DEBUG, "Received new config (" + + if (log.isLoggable(Level.FINE)) { + log.log(Level.FINE, "Received new config (" + "route: " + (defaultRoute != null ? defaultRoute : "<none>") + ", timeout: " + defaultTimeoutMillis + " ms, retry enabled: " + retryEnabled + ", docproc chain: " + (defaultDocprocChain != null ? defaultDocprocChain : "<none>") + diff --git a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java index 4fcbcf4d634..d2e0434f273 100755 --- a/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java +++ b/vespaclient-core/src/main/java/com/yahoo/feedapi/SharedSender.java @@ -2,7 +2,7 @@ package com.yahoo.feedapi; import com.yahoo.concurrent.SystemTimer; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Message; import com.yahoo.messagebus.Reply; @@ -124,22 +124,22 @@ public class SharedSender implements ReplyHandler { synchronized (metrics) { metrics.addReply(r); } - if (log.isLoggable(LogLevel.SPAM)) { - log.log(LogLevel.SPAM, "Received reply for file " + owner.toString() + " count was " + owner.getPending().val()); + if (log.isLoggable(Level.FINEST)) { + log.log(Level.FINEST, "Received reply for file " + owner.toString() + " count was " + owner.getPending().val()); } if (owner.isAborted()) { - log.log(LogLevel.DEBUG, "Received reply for file " + owner.toString() + " which is aborted"); + log.log(Level.FINE, "Received reply for file " + owner.toString() + " which is aborted"); owner.getPending().clear(); return; } if (owner.handleReply(r)) { owner.getPending().dec(); } else { - log.log(LogLevel.DEBUG, "Received reply for file " + owner.toString() + " which wants to abort"); + log.log(Level.FINE, "Received reply for file " + owner.toString() + " which wants to abort"); owner.getPending().clear(); } } else { - log.log(LogLevel.WARNING, "Received reply " + r + " for message " + r.getMessage() + " without context"); + log.log(Level.WARNING, "Received reply " + r + " for message " + r.getMessage() + " without context"); } } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java index d88de7d0fef..bfd8d47eb09 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java @@ -13,7 +13,7 @@ import com.yahoo.documentapi.messagebus.protocol.DocumentListEntry; import com.yahoo.documentapi.messagebus.protocol.DocumentListMessage; import com.yahoo.documentapi.messagebus.protocol.EmptyBucketsMessage; import com.yahoo.documentapi.messagebus.protocol.MapVisitorMessage; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.messagebus.Message; import java.io.IOException; @@ -244,7 +244,7 @@ public class StdOutVisitorHandler extends VdsVisitHandler { buckets.append(" "); buckets.append(bid.toString()); } - log.log(LogLevel.INFO, "Got EmptyBuckets: " + buckets); + log.log(Level.INFO, "Got EmptyBuckets: " + buckets); } @Override diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java index 50f5c9d484c..a1dec70f9df 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitTarget.java @@ -9,7 +9,7 @@ import com.yahoo.documentapi.VisitorDestinationSession; import com.yahoo.documentapi.messagebus.MessageBusDocumentAccess; import com.yahoo.documentapi.messagebus.MessageBusParams; import com.yahoo.documentapi.messagebus.loadtypes.LoadTypeSet; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.log.LogSetup; import com.yahoo.messagebus.network.Identity; import com.yahoo.net.HostName; @@ -208,7 +208,7 @@ public class VdsVisitTarget { @SuppressWarnings("unchecked") public void run() throws Exception { initShutdownHook(); - log.log(LogLevel.DEBUG, "Starting VdsVisitTarget"); + log.log(Level.FINE, "Starting VdsVisitTarget"); MessageBusParams mbusParams = new MessageBusParams(new LoadTypeSet()); mbusParams.getRPCNetworkParams().setIdentity(new Identity(slobrokAddress)); diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index 4090283c10f..042a1f69bb2 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -673,7 +673,7 @@ void generateData(std::vector<LeafPair> & data, size_t numEntries) { data.reserve(numEntries); - Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(10); for (size_t i = 0; i < numEntries; ++i) { int num = rnd.lrand48() % 10000000; diff --git a/vespalib/src/tests/btree/btreeaggregation_test.cpp b/vespalib/src/tests/btree/btreeaggregation_test.cpp index 0ce3d2c7d04..2b4b37af171 100644 --- a/vespalib/src/tests/btree/btreeaggregation_test.cpp +++ b/vespalib/src/tests/btree/btreeaggregation_test.cpp @@ -1,7 +1,5 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("btreeaggregation_test"); -#include <vespa/vespalib/testkit/testapp.h> + #include <vespa/vespalib/btree/btreeroot.h> #include <vespa/vespalib/btree/btreebuilder.h> #include <vespa/vespalib/btree/btreenodeallocator.h> @@ -17,6 +15,7 @@ LOG_SETUP("btreeaggregation_test"); #include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/btree/btreeaggregator.hpp> #include <vespa/vespalib/test/btree/btree_printer.h> +#include <vespa/vespalib/testkit/testapp.h> #include <vespa/vespalib/util/rand48.h> #include <iostream> @@ -24,11 +23,13 @@ LOG_SETUP("btreeaggregation_test"); #include <set> #include <string> +#include <vespa/log/log.h> +LOG_SETUP("btreeaggregation_test"); + using vespalib::GenerationHandler; using search::datastore::EntryRef; -namespace search { -namespace btree { +namespace search::btree { namespace { @@ -99,6 +100,11 @@ typedef std::less<int> MyComp; #define UNWRAP(key) (key) #endif +struct KeyMinMaxAggrCalc : MinMaxAggrCalc { + constexpr static bool aggregate_over_values() { return false; } + constexpr static int32_t getVal(const MyKey& key) { return key._val; } +}; + typedef BTree<MyKey, int32_t, btree::MinMaxAggregated, MyComp, MyTraits, @@ -130,6 +136,7 @@ struct LeafPairLess { } }; +using MyKeyAggrTree = BTree<MyKey, int32_t, btree::MinMaxAggregated, MyComp, MyTraits, KeyMinMaxAggrCalc>; class MockTree { @@ -311,10 +318,13 @@ private: size_t numEntries); void requireThatNodeInsertWorks(); + void keys_are_aggregated_correctly_on_node_insertions(); void requireThatNodeSplitInsertWorks(); + void keys_are_aggregated_correctly_when_node_split_on_insert(); void requireThatTreeInsertWorks(); void requireThatNodeStealWorks(); void requireThatNodeRemoveWorks(); + void keys_are_aggregated_correctly_on_node_removal(); void requireThatWeCanInsertAndRemoveFromTree(); void requireThatSortedTreeInsertWorks(); void requireThatCornerCaseTreeFindWorks(); @@ -322,6 +332,7 @@ private: void requireThatTreeIteratorAssignWorks(); void requireThatUpdateOfKeyWorks(); void requireThatUpdateOfDataWorks(); + void requireThatFrozenViewProvidesAggregatedValues(); template <typename TreeStore> void @@ -409,6 +420,17 @@ Test::requireThatNodeInsertWorks() EXPECT_TRUE(assertTree("{{10:101,20:102,30:103,40:104[min=101,max=104]}}", t)); } +void Test::keys_are_aggregated_correctly_on_node_insertions() { + MyKeyAggrTree t; + t.insert(20, 102); + EXPECT_TRUE(assertTree("{{20:102[min=20,max=20]}}", t)); + t.insert(10, 101); + EXPECT_TRUE(assertTree("{{10:101,20:102[min=10,max=20]}}", t)); + t.insert(30, 103); + t.insert(40, 104); + EXPECT_TRUE(assertTree("{{10:101,20:102,30:103,40:104[min=10,max=40]}}", t)); +} + template <typename Tree> void populateTree(Tree &t, uint32_t count, uint32_t delta) @@ -422,8 +444,9 @@ populateTree(Tree &t, uint32_t count, uint32_t delta) } } +template <typename Tree> void -populateLeafNode(MyTree &t) +populateLeafNode(Tree &t) { populateTree(t, 4, 2); } @@ -457,6 +480,33 @@ Test::requireThatNodeSplitInsertWorks() } } +void Test::keys_are_aggregated_correctly_when_node_split_on_insert() { + { // new entry in current node + MyKeyAggrTree t; + populateLeafNode(t); + t.insert(4, 104); + EXPECT_TRUE(assertTree("{{4,7[min=1,max=7]}} -> " + "{{1:101,3:103,4:104[min=1,max=4]}," + "{5:105,7:107[min=5,max=7]}}", t)); + } + { // new entry in split node + MyKeyAggrTree t; + populateLeafNode(t); + t.insert(6, 106); + EXPECT_TRUE(assertTree("{{5,7[min=1,max=7]}} -> " + "{{1:101,3:103,5:105[min=1,max=5]}," + "{6:106,7:107[min=6,max=7]}}", t)); + } + { // new entry at end + MyKeyAggrTree t; + populateLeafNode(t); + t.insert(8, 108); + EXPECT_TRUE(assertTree("{{5,8[min=1,max=8]}} -> " + "{{1:101,3:103,5:105[min=1,max=5]}," + "{7:107,8:108[min=7,max=8]}}", t)); + } +} + void Test::requireThatTreeInsertWorks() { @@ -645,11 +695,22 @@ Test::requireThatNodeRemoveWorks() EXPECT_TRUE(assertTree("{{5:105[min=105,max=105]}}", t)); } +void Test::keys_are_aggregated_correctly_on_node_removal() { + MyKeyAggrTree t; + populateLeafNode(t); + t.remove(3); + EXPECT_TRUE(assertTree("{{1:101,5:105,7:107[min=1,max=7]}}", t)); + t.remove(1); + EXPECT_TRUE(assertTree("{{5:105,7:107[min=5,max=7]}}", t)); + t.remove(7); + EXPECT_TRUE(assertTree("{{5:105[min=5,max=5]}}", t)); +} + void generateData(std::vector<LeafPair> & data, size_t numEntries) { data.reserve(numEntries); - Rand48 rnd; + vespalib::Rand48 rnd; rnd.srand48(10); for (size_t i = 0; i < numEntries; ++i) { int num = rnd.lrand48() % 10000000; @@ -960,10 +1021,8 @@ struct UpdKeyComp { void Test::requireThatUpdateOfKeyWorks() { - typedef BTree<int, BTreeNoLeafData, - btree::NoAggregated, - UpdKeyComp &> UpdKeyTree; - typedef UpdKeyTree::Iterator UpdKeyTreeIterator; + using UpdKeyTree = BTree<int, BTreeNoLeafData, btree::NoAggregated, UpdKeyComp &>; + using UpdKeyTreeIterator = UpdKeyTree::Iterator; GenerationHandler g; UpdKeyTree t; UpdKeyComp cmp1(0); @@ -1127,6 +1186,24 @@ Test::requireThatSmallNodesWorks() s.trimHoldLists(g.getFirstUsedGeneration()); } +void +Test::requireThatFrozenViewProvidesAggregatedValues() +{ + MyTree t; + t.insert(20, 102); + t.insert(10, 101); + t.insert(30, 103); + t.insert(40, 104); + auto old_view = t.getFrozenView(); + t.getAllocator().freeze(); + auto new_view = t.getFrozenView(); + auto new_aggregated = new_view.getAggregated(); + EXPECT_EQUAL(new_aggregated.getMin(), 101); + EXPECT_EQUAL(new_aggregated.getMax(), 104); + auto old_aggregated = old_view.getAggregated(); + EXPECT_EQUAL(old_aggregated.getMin(), std::numeric_limits<int32_t>::max()); + EXPECT_EQUAL(old_aggregated.getMax(), std::numeric_limits<int32_t>::min()); +} int Test::Main() @@ -1134,10 +1211,13 @@ Test::Main() TEST_INIT("btreeaggregation_test"); requireThatNodeInsertWorks(); + keys_are_aggregated_correctly_on_node_insertions(); requireThatNodeSplitInsertWorks(); + keys_are_aggregated_correctly_when_node_split_on_insert(); requireThatTreeInsertWorks(); requireThatNodeStealWorks(); requireThatNodeRemoveWorks(); + keys_are_aggregated_correctly_on_node_removal(); requireThatWeCanInsertAndRemoveFromTree(); requireThatSortedTreeInsertWorks(); requireThatCornerCaseTreeFindWorks(); @@ -1147,11 +1227,11 @@ Test::Main() requireThatUpdateOfDataWorks(); TEST_DO(requireThatSmallNodesWorks<MyTreeStore>()); TEST_DO(requireThatSmallNodesWorks<MyTreeForceApplyStore>()); + TEST_DO(requireThatFrozenViewProvidesAggregatedValues()); TEST_DONE(); } } -} TEST_APPHOOK(search::btree::Test); diff --git a/vespalib/src/tests/btree/frozenbtree_test.cpp b/vespalib/src/tests/btree/frozenbtree_test.cpp index 597c2ffdc90..9bf6f3773eb 100644 --- a/vespalib/src/tests/btree/frozenbtree_test.cpp +++ b/vespalib/src/tests/btree/frozenbtree_test.cpp @@ -47,7 +47,7 @@ private: NodeAllocator *_allocator; Tree *_tree; - Rand48 _randomGenerator; + vespalib::Rand48 _randomGenerator; void allocTree(); void freeTree(bool verbose); @@ -421,7 +421,9 @@ FrozenBTreeTest::Main() allocTree(); insertRandomValues(*_tree, *_allocator, _randomValues); lookupRandomValues(*_tree, *_allocator, _randomValues); + EXPECT_TRUE(_tree->getFrozenView(*_allocator).empty()); _allocator->freeze(); + EXPECT_FALSE(_tree->getFrozenView(*_allocator).empty()); _allocator->transferHoldLists(_generationHandler->getCurrentGeneration()); lookupFrozenRandomValues(*_tree, *_allocator, _randomValues); traverseTreeIterator(*_tree, diff --git a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp index 6a51b9192ce..3b748d1d4c7 100644 --- a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp +++ b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp @@ -36,19 +36,19 @@ struct Fixture { Setup setup; IntBufferType bufferType; size_t deadElems; - int buffer; + int buffer[ARRAYS_SIZE]; Fixture(const Setup &setup_) : setup(setup_), bufferType(ARRAYS_SIZE, setup._minArrays, MAX_ARRAYS, NUM_ARRAYS_FOR_NEW_BUFFER, setup._allocGrowFactor), deadElems(0), - buffer(0) + buffer() {} ~Fixture() { bufferType.onHold(&setup._usedElems); bufferType.onFree(setup._usedElems); } void onActive() { - bufferType.onActive(setup._bufferId, &setup._usedElems, deadElems, &buffer); + bufferType.onActive(setup._bufferId, &setup._usedElems, deadElems, &buffer[0]); } size_t arraysToAlloc() { return bufferType.calcArraysToAlloc(setup._bufferId, setup._neededElems, setup._resizing); diff --git a/vespalib/src/vespa/vespalib/btree/btreeaggregator.hpp b/vespalib/src/vespa/vespalib/btree/btreeaggregator.hpp index e1318ab5a66..34dbfd3e899 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeaggregator.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeaggregator.hpp @@ -13,7 +13,11 @@ BTreeAggregator<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, AggrCalcT>::aggr { AggrT a; for (uint32_t i = 0, ie = node.validSlots(); i < ie; ++i) { - aggrCalc.add(a, aggrCalc.getVal(node.getData(i))); + if constexpr (AggrCalcT::aggregate_over_values()) { + aggrCalc.add(a, aggrCalc.getVal(node.getData(i))); + } else { + aggrCalc.add(a, aggrCalc.getVal(node.getKey(i))); + } } return a; } diff --git a/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp b/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp index e1518da5639..5c6b02a5a46 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeinserter.hpp @@ -121,7 +121,11 @@ insert(BTreeNode::Ref &root, lnode->insert(idx, key, data); itr.setLeafNodeIdx(idx); if constexpr (AggrCalcT::hasAggregated()) { - aggrCalc.add(lnode->getAggregated(), aggrCalc.getVal(data)); + if constexpr (AggrCalcT::aggregate_over_values()) { + aggrCalc.add(lnode->getAggregated(), aggrCalc.getVal(data)); + } else { + aggrCalc.add(lnode->getAggregated(), aggrCalc.getVal(key)); + } ca = lnode->getAggregated(); } } diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp index 27a50e04f1b..98a14943d36 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp @@ -1089,12 +1089,12 @@ BTreeIterator<KeyT, DataT, AggrT, CompareT, TraitsT>:: updateData(const DataType & data, [[maybe_unused]] const AggrCalcT &aggrCalc) { LeafNodeType * lnode = getLeafNode(); - if constexpr (AggrCalcT::hasAggregated()) { + if constexpr (AggrCalcT::hasAggregated() && AggrCalcT::aggregate_over_values()) { AggrT oldca(lnode->getAggregated()); - typedef BTreeAggregator<KeyT, DataT, AggrT, - TraitsT::INTERNAL_SLOTS, - TraitsT::LEAF_SLOTS, - AggrCalcT> Aggregator; + using Aggregator = BTreeAggregator<KeyT, DataT, AggrT, + TraitsT::INTERNAL_SLOTS, + TraitsT::LEAF_SLOTS, + AggrCalcT>; if (aggrCalc.update(lnode->getAggregated(), aggrCalc.getVal(lnode->getData(_leaf.getIdx())), aggrCalc.getVal(data))) { @@ -1193,7 +1193,11 @@ insertFirst(const KeyType &key, const DataType &data, lnode.data->insert(0, key, data); if constexpr (AggrCalcT::hasAggregated()) { AggrT a; - aggrCalc.add(a, aggrCalc.getVal(data)); + if constexpr (AggrCalcT::aggregate_over_values()) { + aggrCalc.add(a, aggrCalc.getVal(data)); + } else { + aggrCalc.add(a, aggrCalc.getVal(key)); + } lnode.data->getAggregated() = a; } _leafRoot = lnode.data; diff --git a/vespalib/src/vespa/vespalib/btree/btreeremover.hpp b/vespalib/src/vespa/vespalib/btree/btreeremover.hpp index 6a9a7ebcc7e..acb51b18b0c 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeremover.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeremover.hpp @@ -110,11 +110,17 @@ remove(BTreeNode::Ref &root, NodeAllocatorType &allocator(itr.getAllocator()); AggrT oldca(AggrCalcT::hasAggregated() ? lnode->getAggregated() : AggrT()); AggrT ca; - if (AggrCalcT::hasAggregated() && - aggrCalc.remove(lnode->getAggregated(), - aggrCalc.getVal(lnode->getData(idx)))) { + if constexpr (AggrCalcT::hasAggregated()) { + bool need_aggregation_recalc; + if constexpr (AggrCalcT::aggregate_over_values()) { + need_aggregation_recalc = aggrCalc.remove(lnode->getAggregated(), aggrCalc.getVal(lnode->getData(idx))); + } else { + need_aggregation_recalc = aggrCalc.remove(lnode->getAggregated(), aggrCalc.getVal(lnode->getKey(idx))); + } lnode->remove(idx); - Aggregator::recalc(*lnode, aggrCalc); + if (need_aggregation_recalc) { + Aggregator::recalc(*lnode, aggrCalc); + } } else { lnode->remove(idx); } diff --git a/vespalib/src/vespa/vespalib/btree/btreeroot.h b/vespalib/src/vespa/vespalib/btree/btreeroot.h index b5759a6a341..471f4273632 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeroot.h +++ b/vespalib/src/vespa/vespalib/btree/btreeroot.h @@ -83,6 +83,12 @@ public: size_t size() const; const NodeAllocatorType &getAllocator() const { return _allocator; } + const AggrT &getAggregated() const { + return _allocator.getAggregated(_frozenRoot); + } + + bool empty() const { return !_frozenRoot.valid(); } + template <typename FunctionType> void foreach_key(FunctionType func) const { _allocator.getNodeStore().foreach_key(_frozenRoot, func); diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.hpp b/vespalib/src/vespa/vespalib/btree/btreestore.hpp index ec3be588de4..e6c96600b70 100644 --- a/vespalib/src/vespa/vespalib/btree/btreestore.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreestore.hpp @@ -959,7 +959,11 @@ getAggregated(const EntryRef ref) const const KeyDataType *shortArray = getKeyDataEntry(iRef, clusterSize); AggregatedType a; for (uint32_t i = 0; i < clusterSize; ++i) { - _aggrCalc.add(a, _aggrCalc.getVal(shortArray[i].getData())); + if constexpr (AggrCalcT::aggregate_over_values()) { + _aggrCalc.add(a, _aggrCalc.getVal(shortArray[i].getData())); + } else { + _aggrCalc.add(a, _aggrCalc.getVal(shortArray[i].getKey())); + } } return a; } diff --git a/vespalib/src/vespa/vespalib/btree/minmaxaggrcalc.h b/vespalib/src/vespa/vespalib/btree/minmaxaggrcalc.h index 66e23127c5a..6ef809861b7 100644 --- a/vespalib/src/vespa/vespalib/btree/minmaxaggrcalc.h +++ b/vespalib/src/vespa/vespalib/btree/minmaxaggrcalc.h @@ -11,6 +11,7 @@ class MinMaxAggrCalc public: constexpr MinMaxAggrCalc() = default; constexpr static bool hasAggregated() { return true; } + constexpr static bool aggregate_over_values() { return true; } static int32_t getVal(int32_t val) { return val; } static void add(MinMaxAggregated &a, int32_t val) { a.add(val); } static void add(MinMaxAggregated &a, const MinMaxAggregated &ca) { a.add(ca); } diff --git a/vespalib/src/vespa/vespalib/btree/noaggrcalc.h b/vespalib/src/vespa/vespalib/btree/noaggrcalc.h index 079abc57224..51958229430 100644 --- a/vespalib/src/vespa/vespalib/btree/noaggrcalc.h +++ b/vespalib/src/vespa/vespalib/btree/noaggrcalc.h @@ -17,6 +17,8 @@ public: return false; } + constexpr static bool aggregate_over_values() { return true; } + template <typename DataT> static inline int32_t getVal(const DataT &val) diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index 33830bbe4ab..956d4c16d40 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -10,7 +10,6 @@ #include "unique_store_builder.hpp" #include "unique_store_dictionary.hpp" #include "unique_store_enumerator.hpp" -#include <vespa/vespalib/util/bufferwriter.h> #include <atomic> #include <algorithm> diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index e3397b54740..4029c4881c4 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -12,7 +12,6 @@ vespa_add_library(vespalib_vespalib_util OBJECT benchmark_timer.cpp blockingthreadstackexecutor.cpp box.cpp - bufferwriter.cpp classname.cpp closuretask.cpp compress.cpp diff --git a/vespalib/src/vespa/vespalib/util/rand48.h b/vespalib/src/vespa/vespalib/util/rand48.h index 441f8e6e10f..3ebfe00ba35 100644 --- a/vespalib/src/vespa/vespalib/util/rand48.h +++ b/vespalib/src/vespa/vespalib/util/rand48.h @@ -2,8 +2,10 @@ #pragma once +#include <cstdlib> +#include <cstdint> -namespace search { +namespace vespalib { /* * Simple random generator based on lrand48() spec. @@ -19,19 +21,19 @@ public: _state = ((static_cast<uint64_t>(seed & 0xffffffffu)) << 16) + 0x330e; } - Rand48(void) + Rand48() : _state(0) { srand48(0x1234abcd); }; - void iterate(void) { + void iterate() { _state = (UINT64_C(0x5DEECE66D) * _state + 0xb) & UINT64_C(0xFFFFFFFFFFFF); } /* * Return value from 0 to 2^31 - 1 */ - long lrand48(void) { + long lrand48() { iterate(); return static_cast<long>(_state >> 17); } diff --git a/vespalib/src/vespa/vespalib/util/time.cpp b/vespalib/src/vespa/vespalib/util/time.cpp index 61a295c6dc3..341102bf9ff 100644 --- a/vespalib/src/vespa/vespalib/util/time.cpp +++ b/vespalib/src/vespa/vespalib/util/time.cpp @@ -53,6 +53,8 @@ Timer::waitAtLeast(duration dur, bool busyWait) { } +#ifndef __clang__ + namespace std::chrono { /* @@ -83,3 +85,4 @@ steady_clock::now() noexcept { } } +#endif diff --git a/vespamalloc/src/tests/allocfree/CMakeLists.txt b/vespamalloc/src/tests/allocfree/CMakeLists.txt index 52c0c03d079..34d402d0c60 100644 --- a/vespamalloc/src/tests/allocfree/CMakeLists.txt +++ b/vespamalloc/src/tests/allocfree/CMakeLists.txt @@ -21,7 +21,7 @@ vespa_add_executable(vespamalloc_linklist_test_app $<TARGET_OBJECTS:vespamalloc_util> DEPENDS dl - atomic + ${VESPA_ATOMIC_LIB} ) vespa_add_test(NAME vespamalloc_allocfree_shared_test_app NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/allocfree_test.sh BENCHMARK DEPENDS vespamalloc_realloc_test_app vespamalloc_allocfree_shared_test_app vespamalloc_linklist_test_app diff --git a/vespamalloc/src/tests/test1/CMakeLists.txt b/vespamalloc/src/tests/test1/CMakeLists.txt index cbfb9543cda..cade2e092b4 100644 --- a/vespamalloc/src/tests/test1/CMakeLists.txt +++ b/vespamalloc/src/tests/test1/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(vespamalloc_testatomic_app TEST SOURCES testatomic.cpp DEPENDS - atomic + ${VESPA_ATOMIC_LIB} ) vespa_add_test(NAME vespamalloc_testatomic_app NO_VALGRIND COMMAND vespamalloc_testatomic_app) diff --git a/vespamalloc/src/vespamalloc/CMakeLists.txt b/vespamalloc/src/vespamalloc/CMakeLists.txt index ac7fd300828..1fe550ad95a 100644 --- a/vespamalloc/src/vespamalloc/CMakeLists.txt +++ b/vespamalloc/src/vespamalloc/CMakeLists.txt @@ -5,7 +5,7 @@ vespa_add_library(vespamalloc $<TARGET_OBJECTS:vespamalloc_util> INSTALL lib64/vespa/malloc DEPENDS - atomic + ${VESPA_ATOMIC_LIB} dl ) vespa_add_library(vespamallocd @@ -14,7 +14,7 @@ vespa_add_library(vespamallocd $<TARGET_OBJECTS:vespamalloc_util> INSTALL lib64/vespa/malloc DEPENDS - atomic + ${VESPA_ATOMIC_LIB} dl ) vespa_add_library(vespamallocdst16 @@ -23,7 +23,7 @@ vespa_add_library(vespamallocdst16 $<TARGET_OBJECTS:vespamalloc_util> INSTALL lib64/vespa/malloc DEPENDS - atomic + ${VESPA_ATOMIC_LIB} dl ) vespa_add_library(vespamallocdst16_nl @@ -32,7 +32,7 @@ vespa_add_library(vespamallocdst16_nl $<TARGET_OBJECTS:vespamalloc_util> INSTALL lib64/vespa/malloc DEPENDS - atomic + ${VESPA_ATOMIC_LIB} dl ) vespa_add_library(vespammap diff --git a/zkfacade/pom.xml b/zkfacade/pom.xml index b857c31d262..de542b89b67 100644 --- a/zkfacade/pom.xml +++ b/zkfacade/pom.xml @@ -72,12 +72,6 @@ <version>1.7.5</version> </dependency> <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>vespalog</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> <version>${zookeeper.client.version}</version> diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java index f9bece33a39..d71e23a0623 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/CuratorCompletionWaiter.java @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.curator; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.path.Path; import org.apache.curator.framework.CuratorFramework; @@ -39,9 +39,9 @@ class CuratorCompletionWaiter implements Curator.CompletionWaiter { public void awaitCompletion(Duration timeout) { List<String> respondents; try { - log.log(LogLevel.DEBUG, "Synchronizing on barrier " + barrierPath); + log.log(Level.FINE, "Synchronizing on barrier " + barrierPath); respondents = awaitInternal(timeout); - log.log(LogLevel.DEBUG, "Done synchronizing on barrier " + barrierPath); + log.log(Level.FINE, "Done synchronizing on barrier " + barrierPath); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java index b8a96d916f5..1f4be8852a3 100644 --- a/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ b/zookeeper-server/zookeeper-server-3.5/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.zookeeper; import com.google.inject.Inject; import com.yahoo.cloud.config.ZookeeperServerConfig; import com.yahoo.component.AbstractComponent; -import com.yahoo.log.LogLevel; +import java.util.logging.Level; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyStoreUtils; @@ -165,7 +165,7 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna try { zkServerThread.join(); } catch (InterruptedException e) { - log.log(LogLevel.WARNING, "Error joining server thread on shutdown", e); + log.log(Level.WARNING, "Error joining server thread on shutdown", e); } } @@ -173,7 +173,7 @@ public class VespaZooKeeperServerImpl extends AbstractComponent implements Runna public void run() { System.setProperty(ZOOKEEPER_JMX_LOG4J_DISABLE, "true"); String[] args = new String[]{getDefaults().underVespaHome(zookeeperServerConfig.zooKeeperConfigFile())}; - log.log(LogLevel.INFO, "Starting ZooKeeper server with config file " + args[0] + + log.log(Level.INFO, "Starting ZooKeeper server with config file " + args[0] + ". Trying to establish ZooKeeper quorum (members: " + zookeeperServerHostnames(zookeeperServerConfig) + ")"); org.apache.zookeeper.server.quorum.QuorumPeerMain.main(args); } |