diff options
84 files changed, 1225 insertions, 1005 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java index 711a3d1e72b..bb0d24f9b26 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java @@ -14,7 +14,6 @@ import org.w3c.dom.Element; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.List; -import java.util.Optional; /** * Builds a config model using DOM parsers diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 063c295e33c..442d6bbf368 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.container; import com.yahoo.component.ComponentId; import com.yahoo.component.ComponentSpecification; -import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.container.ComponentsConfig; import com.yahoo.container.QrConfig; @@ -39,8 +38,8 @@ import static com.yahoo.container.QrConfig.Rpc; /** * @author gjoranv - * @author einarmr - * @author tonytv + * @author Einar M R Rosenvinge + * @author Tony Vaagenes */ //qr is restart because it is handled by ConfiguredApplication.start @RestartConfigs({QrStartConfig.class, QrConfig.class}) @@ -59,14 +58,8 @@ public class Container extends AbstractService implements private final boolean isHostedVespa; private String clusterName = null; - private boolean rpcServerEnabled = true; - private Optional<String> hostResponseHeaderKey = Optional.empty(); - // TODO: move these up to cluster - private boolean httpServerEnabled = true; - private boolean messageBusEnabled = true; - /** Whether this node has been marked as retired (e.g, will be removed) */ private final boolean retired; /** The unique index of this node */ @@ -84,10 +77,10 @@ public class Container extends AbstractService implements private static final String defaultHostedJVMArgs = "-XX:+UseOSErrorReporting -XX:+SuppressFatalErrorMessage"; public Container(AbstractConfigProducer parent, String name, int index) { - this(parent, name, Collections.<PortOverride>emptyList(), index); + this(parent, name, Collections.emptyList(), index); } public Container(AbstractConfigProducer parent, String name, boolean retired, int index) { - this(parent, name, retired, Collections.<PortOverride>emptyList(), index); + this(parent, name, retired, Collections.emptyList(), index); } public Container(AbstractConfigProducer parent, String name, List<PortOverride> portOverrides, int index) { this(parent, name, false, portOverrides, index); @@ -159,6 +152,8 @@ public class Container extends AbstractService implements @Override public void initService() { + if (isInitialized()) return; + // XXX: Must be called first, to set the baseport super.initService(); @@ -179,7 +174,7 @@ public class Container extends AbstractService implements for (int i = 1; i < numHttpServerPorts; i++) portsMeta.on(i).tag("http").tag("external"); - if (rpcServerEnabled) { + if (rpcServerEnabled()) { portsMeta.on(numHttpServerPorts + 0).tag("rpc").tag("messaging"); portsMeta.on(numHttpServerPorts + 1).tag("rpc").tag("admin"); } @@ -252,7 +247,7 @@ public class Container extends AbstractService implements */ public int getPortCount() { int httpPorts = (getHttp() != null) ? 0 : numHttpServerPorts + 2; // TODO remove +2, only here to keep irrelevant unit tests from failing. - int rpcPorts = (isRpcServerEnabled()) ? numRpcServerPorts : 0; + int rpcPorts = (rpcServerEnabled()) ? numRpcServerPorts : 0; return httpPorts + rpcPorts; } @@ -268,7 +263,7 @@ public class Container extends AbstractService implements } private int getRpcPort() { - return isRpcServerEnabled() ? getRelativePort(numHttpServerPorts + 1) : 0; + return rpcServerEnabled() ? getRelativePort(numHttpServerPorts + 1) : 0; } private int getMessagingPort() { @@ -286,7 +281,7 @@ public class Container extends AbstractService implements return getRelativePort(0); } } else { - return httpServerEnabled ? getSearchPort() : -1; + return httpServerEnabled() ? getSearchPort() : -1; } } @@ -294,15 +289,11 @@ public class Container extends AbstractService implements return "PRELOAD=" + getPreLoad() + " exec vespa-start-container-daemon " + getJvmArgs() + " "; } - public boolean isRpcServerEnabled() { - return rpcServerEnabled; - } - @Override public void getConfig(QrConfig.Builder builder) { builder. rpc(new Rpc.Builder() - .enabled(isRpcServerEnabled()) + .enabled(rpcServerEnabled()) .port(getRpcPort()) .slobrokId(serviceSlobrokId())). filedistributor(filedistributorConfig()); @@ -352,7 +343,7 @@ public class Container extends AbstractService implements private void addAllEnabledComponents(Collection<Component<?, ?>> allComponents, AbstractConfigProducer<?> current) { for (AbstractConfigProducer<?> child: current.getChildren().values()) { - if ( ! httpServerEnabled && isHttpServer(child)) continue; + if ( ! httpServerEnabled() && isHttpServer(child)) continue; if (child instanceof Component) allComponents.add((Component<?, ?>) child); @@ -378,7 +369,7 @@ public class Container extends AbstractService implements @Override public void getConfig(ContainerMbusConfig.Builder builder) { - builder.enabled(messageBusEnabled).port(getMessagingPort()); + builder.enabled(messageBusEnabled()).port(getMessagingPort()); } @Override @@ -389,14 +380,17 @@ public class Container extends AbstractService implements return dimensions; } - public void setRpcServerEnabled(boolean rpcServerEnabled) { - this.rpcServerEnabled = rpcServerEnabled; + private boolean messageBusEnabled() { + return containerCluster().isPresent() && containerCluster().get().messageBusEnabled(); } - public void setHttpServerEnabled(boolean httpServerEnabled) { - this.httpServerEnabled = httpServerEnabled; + private boolean httpServerEnabled() { + return containerCluster().isPresent() && containerCluster().get().httpServerEnabled(); } + private boolean rpcServerEnabled() { + return containerCluster().isPresent() && containerCluster().get().rpcServerEnabled(); + } public static final class PortOverride { public final ComponentSpecification serverId; @@ -408,4 +402,8 @@ public class Container extends AbstractService implements } } + private Optional<ContainerCluster> containerCluster() { + return (parent instanceof ContainerCluster) ? Optional.of((ContainerCluster) parent) : Optional.empty(); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 6ee525c1246..a627525ab77 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -14,7 +14,6 @@ import com.yahoo.config.docproc.SchemamappingConfig; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; -import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.provision.Zone; import com.yahoo.container.BundlesConfig; import com.yahoo.container.ComponentsConfig; @@ -152,6 +151,8 @@ public final class ContainerCluster public static final String ROOT_HANDLER_BINDING = "*://*/"; + private static final boolean messageBusEnabled = true; + private final String name; private List<Container> containers = new ArrayList<>(); @@ -164,6 +165,8 @@ public final class ContainerCluster private SecretStore secretStore; private MbusParams mbusParams; + private boolean rpcServerEnabled = true; + private boolean httpServerEnabled = true; private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); private final Set<Path> platformBundles = new LinkedHashSet<>(); @@ -805,6 +808,16 @@ public final class ContainerCluster */ public Optional<Integer> getMemoryPercentage() { return memoryPercentage; } + boolean messageBusEnabled() { return messageBusEnabled; } + + public void setRpcServerEnabled(boolean rpcServerEnabled) { this.rpcServerEnabled = rpcServerEnabled; } + + boolean rpcServerEnabled() { return rpcServerEnabled; } + + boolean httpServerEnabled() { return httpServerEnabled; } + + public void setHttpServerEnabled(boolean httpServerEnabled) { this.httpServerEnabled = httpServerEnabled; } + @Override public String toString() { return "container cluster '" + getName() + "'"; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 39aba9cfdf9..372b3b146a1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -71,7 +71,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; -import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -87,13 +86,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { /** * Path to vip status file for container in Hosted Vespa. Only used if set, else use HOSTED_VESPA_STATUS_FILE */ - static final String HOSTED_VESPA_STATUS_FILE_INSTALL_SETTING = "cloudconfig_server__tenant_vip_status_file"; + private static final String HOSTED_VESPA_STATUS_FILE_INSTALL_SETTING = "cloudconfig_server__tenant_vip_status_file"; public enum Networking { disable, enable } private ApplicationPackage app; private final boolean standaloneBuilder; private final Networking networking; + private final boolean rpcServerEnabled; + private final boolean httpServerEnabled; protected DeployLogger log; public static final List<ConfigModelId> configModelIds = @@ -102,12 +103,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private static final String xmlRendererId = RendererRegistry.xmlRendererId.getName(); private static final String jsonRendererId = RendererRegistry.jsonRendererId.getName(); - private static final Logger logger = Logger.getLogger(ContainerModelBuilder.class.getName()); - public ContainerModelBuilder(boolean standaloneBuilder, Networking networking) { super(ContainerModel.class); this.standaloneBuilder = standaloneBuilder; this.networking = networking; + // Always disable rpc server for standalone container + this.rpcServerEnabled = !standaloneBuilder; + this.httpServerEnabled = networking == Networking.enable; } @Override @@ -124,6 +126,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { ContainerCluster cluster = createContainerCluster(spec, modelContext); addClusterContent(cluster, spec, modelContext); addBundlesForPlatformComponents(cluster); + cluster.setRpcServerEnabled(rpcServerEnabled); + cluster.setHttpServerEnabled(httpServerEnabled); model.setCluster(cluster); } @@ -159,7 +163,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addDefaultHandlers(cluster); addStatusHandlers(cluster, context); - addDefaultComponents(cluster); setDefaultMetricConsumerFactory(cluster); addHttp(spec, cluster); @@ -241,9 +244,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addConfiguredComponents(cluster, spec, "component"); } - protected void addDefaultComponents(ContainerCluster cluster) { - } - protected void setDefaultMetricConsumerFactory(ContainerCluster cluster) { cluster.setDefaultMetricConsumerFactory(MetricDefaultsConfig.Factory.Enum.STATE_MONITOR); } @@ -442,8 +442,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Container node = new Container(cluster, "container.0", 0); HostResource host = allocateSingleNodeHost(cluster, log, containerElement, context); node.setHostResource(host); - if ( ! node.isInitialized() ) // TODO: Fold this into initService - node.initService(); + node.initService(); cluster.addContainers(Collections.singleton(node)); } else { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index c2b4d2f9e62..a73fc95eb05 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -33,7 +33,6 @@ import java.util.Set; * and other queries against the model. * * @author Harald Musum - * @since 2010-12-08 */ public class Application implements ModelResult { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java index a55c05803b7..b4346498f0c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; import com.yahoo.config.model.api.HostInfo; +import com.yahoo.config.model.api.Model; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Version; @@ -33,27 +34,39 @@ public final class ApplicationSet { latestVersion = applications.keySet().stream().max((a, b) -> a.compareTo(b)).get(); } + /** + * Returns the specified version, or the latest if not specified, or if the given version is not + * available and the latest is a permissible substitution. + * + * @throws VersionDoesNotExistException if the specified version is not available and the latest version is not + * a permissible substitution + */ public Application getForVersionOrLatest(Optional<Version> optionalVersion, Instant now) { - return resolveForVersion(optionalVersion.orElse(latestVersion), now); + Version version = optionalVersion.orElse(latestVersion); + return resolveForVersion(version, now) + .orElseThrow(() -> new VersionDoesNotExistException(applicationId + " has no model for Vespa version " + version)); } - private Application resolveForVersion(Version vespaVersion, Instant now) { + private Optional<Application> resolveForVersion(Version vespaVersion, Instant now) { Application application = applications.get(vespaVersion); if (application != null) - return application; + return Optional.of(application); // Does the latest version specify we can use it regardless? Application latest = applications.get(latestVersion); if (latest.getModel().allowModelVersionMismatch(now)) - return latest; + return Optional.of(latest); - throw new VersionDoesNotExistException("No application with Vespa version " + vespaVersion + " exists"); + return Optional.empty(); } - public ApplicationId getId() { - return applicationId; + /** Returns the application for the given version, if available */ + public Optional<Application> get(Version version) { + return Optional.ofNullable(applications.get(version)); } + public ApplicationId getId() { return applicationId; } + public static ApplicationSet fromList(List<Application> applications) { return new ApplicationSet(applications); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 2f05c1a2259..1aaddfce6fc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -74,19 +74,13 @@ public class ModelContextImpl implements ModelContext { } @Override - public ApplicationPackage applicationPackage() { - return applicationPackage; - } + public ApplicationPackage applicationPackage() { return applicationPackage; } @Override - public Optional<Model> previousModel() { - return previousModel; - } + public Optional<Model> previousModel() { return previousModel; } @Override - public Optional<ApplicationPackage> permanentApplicationPackage() { - return permanentApplicationPackage; - } + public Optional<ApplicationPackage> permanentApplicationPackage() { return permanentApplicationPackage; } /** * Returns the host provisioner to use, or empty to use the default provisioner, @@ -94,34 +88,22 @@ public class ModelContextImpl implements ModelContext { */ // TODO: Don't allow empty here but create the right provisioner when this is set up instead @Override - public Optional<HostProvisioner> hostProvisioner() { - return hostProvisioner; - } + public Optional<HostProvisioner> hostProvisioner() { return hostProvisioner; } @Override - public DeployLogger deployLogger() { - return deployLogger; - } + public DeployLogger deployLogger() { return deployLogger; } @Override - public ConfigDefinitionRepo configDefinitionRepo() { - return configDefinitionRepo; - } + public ConfigDefinitionRepo configDefinitionRepo() { return configDefinitionRepo; } @Override - public FileRegistry getFileRegistry() { - return fileRegistry; - } + public FileRegistry getFileRegistry() { return fileRegistry; } @Override - public ModelContext.Properties properties() { - return properties; - } + public ModelContext.Properties properties() { return properties; } @Override - public Optional<File> appDir() { - return appDir; - } + public Optional<File> appDir() { return appDir; } @Override public Version modelVespaVersion() { return modelVespaVersion; } @@ -129,9 +111,6 @@ public class ModelContextImpl implements ModelContext { @Override public Version wantedNodeVespaVersion() { return wantedNodeVespaVersion; } - /** - * @author Ulf Lilleengen - */ public static class Properties implements ModelContext.Properties { private final ApplicationId applicationId; @@ -159,34 +138,22 @@ public class ModelContextImpl implements ModelContext { } @Override - public boolean multitenant() { - return multitenant; - } + public boolean multitenant() { return multitenant; } @Override - public ApplicationId applicationId() { - return applicationId; - } + public ApplicationId applicationId() { return applicationId; } @Override - public List<ConfigServerSpec> configServerSpecs() { - return configServerSpecs; - } + public List<ConfigServerSpec> configServerSpecs() { return configServerSpecs; } @Override - public HostName loadBalancerName() { - return loadBalancerName; - } + public HostName loadBalancerName() { return loadBalancerName; } @Override - public boolean hostedVespa() { - return hostedVespa; - } + public boolean hostedVespa() { return hostedVespa; } @Override - public Zone zone() { - return zone; - } + public Zone zone() { return zone; } @Override public Set<Rotation> rotations() { return rotations; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java index d7b84d75b5c..0d9346101e9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/PreparedModelsBuilder.java @@ -15,6 +15,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.Version; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.host.HostValidator; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; @@ -89,17 +90,14 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P FileDistributionProvider fileDistributionProvider = fileDistributionFactory.createProvider(context.getServerDBSessionDir()); // Use empty on non-hosted systems, use already allocated hosts if available, create connection to a host provisioner otherwise - Optional<HostProvisioner> hostProvisioner = createHostProvisioner(allocatedHosts); - Optional<Model> previousModel = currentActiveApplicationSet - .map(set -> set.getForVersionOrLatest(Optional.of(modelVersion), now).getModel()); ModelContext modelContext = new ModelContextImpl( applicationPackage, - previousModel, + modelOf(modelVersion), permanentApplicationPackage.applicationPackage(), logger, configDefinitionRepo, fileDistributionProvider.getFileRegistry(), - hostProvisioner, + createHostProvisioner(allocatedHosts), properties, getAppDir(applicationPackage), new com.yahoo.component.Version(modelVersion.toString()), @@ -112,6 +110,11 @@ public class PreparedModelsBuilder extends ModelsBuilder<PreparedModelsBuilder.P return new PreparedModelsBuilder.PreparedModelResult(modelVersion, result.getModel(), fileDistributionProvider, result.getConfigChangeActions()); } + private Optional<Model> modelOf(Version version) { + if ( ! currentActiveApplicationSet.isPresent()) return Optional.empty(); + return currentActiveApplicationSet.get().get(version).map(Application::getModel); + } + // This method is an excellent demonstration of what happens when one is too liberal with Optional // -bratseth, who had to write the below :-\ private Optional<HostProvisioner> createHostProvisioner(Optional<AllocatedHosts> allocatedHosts) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java index def78a3d9d8..f260fade41c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionFactoryImpl.java @@ -55,7 +55,6 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { private final Clock clock; public SessionFactoryImpl(GlobalComponentRegistry globalComponentRegistry, - SessionCounter sessionCounter, TenantApplications applicationRepo, TenantFileSystemDirs tenantFileSystemDirs, HostValidator<ApplicationId> hostRegistry, @@ -65,7 +64,7 @@ public class SessionFactoryImpl implements SessionFactory, LocalSessionLoader { this.sessionPreparer = globalComponentRegistry.getSessionPreparer(); this.curator = globalComponentRegistry.getCurator(); this.configCurator = globalComponentRegistry.getConfigCurator(); - this.sessionCounter = sessionCounter; + this.sessionCounter = new SessionCounter(globalComponentRegistry.getConfigCurator(), tenant);; this.sessionsPath = TenantRepository.getSessionsPath(tenant); this.applicationRepo = applicationRepo; this.tenantFileSystemDirs = tenantFileSystemDirs; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index 95a507ede21..cd437704f9a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.config.server.application.ZKTenantApplications; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.monitoring.Metrics; import com.yahoo.vespa.config.server.session.*; -import com.yahoo.vespa.config.server.zookeeper.SessionCounter; import com.yahoo.vespa.defaults.Defaults; import java.io.File; @@ -39,7 +38,6 @@ public class TenantBuilder { private SessionFactory sessionFactory; private LocalSessionLoader localSessionLoader; private TenantApplications applicationRepo; - private SessionCounter sessionCounter; private ReloadHandler reloadHandler; private RequestHandler requestHandler; private RemoteSessionFactory remoteSessionFactory; @@ -81,11 +79,6 @@ public class TenantBuilder { return this; } - public TenantBuilder withReloadHandler(ReloadHandler reloadHandler) { - this.reloadHandler = reloadHandler; - return this; - } - /** * Create a real tenant from the properties given by this builder. * @@ -96,7 +89,6 @@ public class TenantBuilder { createApplicationRepo(); createRemoteSessionFactory(componentRegistry.getClock()); createRemoteSessionRepo(); - createSessionCounter(); createServerDbDirs(); createSessionFactory(); createLocalSessionRepo(); @@ -120,8 +112,8 @@ public class TenantBuilder { private void createSessionFactory() { if (sessionFactory == null || localSessionLoader == null) { - SessionFactoryImpl impl = new SessionFactoryImpl(componentRegistry, sessionCounter, - applicationRepo, tenantFileSystemDirs, hostValidator, tenant); + SessionFactoryImpl impl = new SessionFactoryImpl(componentRegistry, applicationRepo, + tenantFileSystemDirs, hostValidator, tenant); if (sessionFactory == null) { sessionFactory = impl; } @@ -137,12 +129,6 @@ public class TenantBuilder { } } - private void createSessionCounter() { - if (sessionCounter == null) { - sessionCounter = new SessionCounter(componentRegistry.getConfigCurator(), tenant); - } - } - private void createTenantRequestHandler() { if (requestHandler == null || reloadHandler == null) { TenantRequestHandler impl = new TenantRequestHandler(componentRegistry.getMetrics(), @@ -156,9 +142,7 @@ public class TenantBuilder { if (requestHandler == null) { requestHandler = impl; } - if (reloadHandler == null) { - reloadHandler = impl; - } + reloadHandler = impl; } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index 06c64acab33..0a193f7eedd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -17,7 +17,6 @@ import com.yahoo.jdisc.Response; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.GlobalComponentRegistry; -import com.yahoo.vespa.config.server.MockReloadHandler; import com.yahoo.vespa.config.server.SuperModelGenerationCounter; import com.yahoo.vespa.config.server.TestComponentRegistry; import com.yahoo.vespa.config.server.application.ApplicationConvergenceChecker; @@ -83,23 +82,15 @@ public class ApplicationHandlerTest { public void setup() { TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); tenantRepository = new TenantRepository(componentRegistry, false); - - TenantBuilder tenantBuilder1 = TenantBuilder.create(componentRegistry, mytenantName) - .withReloadHandler(new MockReloadHandler()); - tenantRepository.addTenant(tenantBuilder1); - - TenantBuilder tenantBuilder2 = TenantBuilder.create(componentRegistry, foobar) - .withReloadHandler(new MockReloadHandler()); - tenantRepository.addTenant(tenantBuilder2); - + tenantRepository.addTenant(TenantBuilder.create(componentRegistry, mytenantName)); + tenantRepository.addTenant(TenantBuilder.create(componentRegistry, foobar)); provisioner = new SessionHandlerTest.MockProvisioner(); - mockHandler = createMockApplicationHandler( - provisioner, - new ApplicationConvergenceChecker(stateApiFactory), - mockHttpProxy); - listApplicationsHandler = new ListApplicationsHandler( - ListApplicationsHandler.testOnlyContext(), - tenantRepository, Zone.defaultZone()); + mockHandler = createMockApplicationHandler(provisioner, + new ApplicationConvergenceChecker(stateApiFactory), + mockHttpProxy); + listApplicationsHandler = new ListApplicationsHandler(ListApplicationsHandler.testOnlyContext(), + tenantRepository, + Zone.defaultZone()); } private ApplicationHandler createMockApplicationHandler( diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index 537ab5a9f3e..0ac08bfcfb0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -40,8 +40,7 @@ public class HostHandlerTest { public void setup() { TestComponentRegistry componentRegistry = new TestComponentRegistry.Builder().build(); tenantRepository = new TenantRepository(componentRegistry, false); - TenantBuilder tb = TenantBuilder.create(componentRegistry, mytenant) - .withReloadHandler(new MockReloadHandler()); + TenantBuilder tb = TenantBuilder.create(componentRegistry, mytenant); tenantRepository.addTenant(tb); handler = createHostHandler(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java index b036dc4400e..6e56d3c30c3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java @@ -26,10 +26,8 @@ public class TenantHandlerTest extends TenantTest { private final TenantName a = TenantName.from("a"); @Before - public void setup() throws Exception { - handler = new TenantHandler( - TenantHandler.testOnlyContext(), - tenantRepository); + public void setup() { + handler = new TenantHandler(TenantHandler.testOnlyContext(), tenantRepository); } @Test @@ -50,7 +48,7 @@ public class TenantHandlerTest extends TenantTest { } @Test(expected=NotFoundException.class) - public void testGetNonExisting() throws Exception { + public void testGetNonExisting() { handler.handleGET(HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/x", Method.GET)); } @@ -76,7 +74,7 @@ public class TenantHandlerTest extends TenantTest { } @Test - public void testDelete() throws IOException, InterruptedException { + public void testDelete() throws IOException { putSync(HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/a", Method.PUT)); assertEquals(tenantRepository.getTenant(a).getName(), a); TenantDeleteResponse delResp = (TenantDeleteResponse) handler.handleDELETE( @@ -86,7 +84,7 @@ public class TenantHandlerTest extends TenantTest { } @Test - public void testDeleteTenantWithActiveApplications() throws Exception { + public void testDeleteTenantWithActiveApplications() { putSync(HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/" + a, Method.PUT)); Tenant tenant = tenantRepository.getTenant(a); assertEquals(a, tenant.getName()); @@ -109,7 +107,7 @@ public class TenantHandlerTest extends TenantTest { } @Test(expected=BadRequestException.class) - public void testIllegalNameSlashes() throws InterruptedException { + public void testIllegalNameSlashes() { putSync(HttpRequest.createTestRequest("http://deploy.example.yahoo.com:80/application/v2/tenant/a/b", Method.PUT)); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java index 040c28b8f96..7814266f815 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantTest.java @@ -16,7 +16,7 @@ import org.junit.Before; /** * Supertype for tests in the multi tenant application API * - * @author vegardh + * @author Vegard Havdal * */ public class TenantTest extends TestWithCurator { @@ -24,29 +24,20 @@ public class TenantTest extends TestWithCurator { protected TenantRepository tenantRepository; @Before - public void setupTenants() throws Exception { + public void setupTenants() { tenantRepository = createTenants(); } @After - public void closeTenants() throws IOException { + public void closeTenants() { tenantRepository.close(); } - protected TenantRepository createTenants() throws Exception { + private TenantRepository createTenants() { return new TenantRepository(new TestComponentRegistry.Builder().curator(curator).build()); } - protected Executor testExecutor() { - return new Executor() { - @Override - public void execute(Runnable command) { - command.run(); - } - }; - } - - protected void assertResponseEquals(SessionResponse response, String payload) throws IOException { + void assertResponseEquals(SessionResponse response, String payload) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); assertEquals(baos.toString("UTF-8"), payload); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java index 2d182f03de7..73caf770512 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionRepoTest.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.io.IOUtils; import com.yahoo.vespa.config.server.host.HostRegistry; import com.yahoo.vespa.config.server.http.SessionHandlerTest; -import com.yahoo.vespa.config.server.zookeeper.SessionCounter; import org.junit.Before; import org.junit.Test; @@ -50,10 +49,9 @@ public class LocalSessionRepoTest extends TestWithCurator { } clock = new ManualClock(Instant.ofEpochSecond(1)); LocalSessionLoader loader = new SessionFactoryImpl(globalComponentRegistry, - new SessionCounter(globalComponentRegistry.getConfigCurator(), tenantName), - new MemoryTenantApplications(), - tenantFileSystemDirs, new HostRegistry<>(), - tenantName); + new MemoryTenantApplications(), + tenantFileSystemDirs, new HostRegistry<>(), + tenantName); repo = new LocalSessionRepo(tenantFileSystemDirs, loader, clock, 5); } diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 912ccf7e963..fc03f362b88 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -435,7 +435,7 @@ <properties> <bouncycastle.version>1.58</bouncycastle.version> - <felix.version>4.6.1</felix.version> + <felix.version>5.0.1</felix.version> <findbugs.version>1.3.9</findbugs.version> <guava.version>18.0</guava.version> <guice.version>3.0</guice.version> diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java index bb6a033a4ab..9762d69ec31 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/athenz/AthenzIdentityProvider.java @@ -10,4 +10,7 @@ public interface AthenzIdentityProvider { String domain(); String service(); SSLContext getIdentitySslContext(); + SSLContext getRoleSslContext(String domain, String role); + String getRoleToken(String domain); + String getRoleToken(String domain, String role); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 78ea99b523d..e088a37e632 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -48,7 +48,6 @@ import static com.yahoo.vespa.hosted.controller.api.integration.BuildService.Job import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest; import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest; -import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; @@ -127,13 +126,15 @@ public class DeploymentTrigger { report.jobType(), triggering.completion(report.buildNumber(), clock.instant()), report.jobError()); - application = application.withChange(remainingChange(application)); - applications().store(application); }); } + public Map<JobType, ? extends List<? extends BuildJob>> jobsToRun() { + return computeReadyJobs().collect(groupingBy(Job::jobType)); + } + /** * Finds and triggers jobs that can and should run but are currently not, and returns the number of triggered jobs. * @@ -168,7 +169,6 @@ public class DeploymentTrigger { */ public boolean trigger(Job job) { log.log(LogLevel.INFO, String.format("Attempting to trigger %s: %s", job, job.triggering)); - try { buildService.trigger(job); applications().lockOrThrow(job.applicationId(), application -> @@ -184,6 +184,21 @@ public class DeploymentTrigger { } } + public List<JobType> forceTrigger(ApplicationId applicationId, JobType jobType) { + Application application = applications().require(applicationId); + if (jobType == component) { + buildService.trigger(BuildJob.of(applicationId, application.deploymentJobs().projectId().getAsLong(), jobType.jobName())); + return singletonList(component); + } + Versions versions = versionsFor(application, application.change(), deploymentFor(application, jobType)); + String reason = ">:o:< Triggered by force! (-o-) |-o-| (=oo=)"; + return (jobType.isProduction() && ! isTested(application, versions) + ? testJobsFor(application, versions, reason, clock.instant()).stream() + : Stream.of(deploymentJob(application, versions, application.change(), jobType, reason, clock.instant()))) + .peek(this::trigger) + .map(Job::jobType).collect(toList()); + } + /** * Triggers a change of this application * @@ -213,64 +228,27 @@ public class DeploymentTrigger { }); } - public Map<JobType, ? extends List<? extends BuildJob>> jobsToRun() { - return computeReadyJobs().collect(groupingBy(Job::jobType)); - } + // ---------- Conveniences ---------- - public List<JobType> forceTrigger(ApplicationId applicationId, JobType jobType) { - Application application = applications().require(applicationId); - if (jobType == component) { - buildService.trigger(BuildJob.of(applicationId, application.deploymentJobs().projectId().getAsLong(), jobType.jobName())); - return singletonList(component); - } - Versions versions = versionsFor(application, application.change(), deploymentFor(application, jobType)); - String reason = ">:o:< Triggered by force! (-o-) |-o-| (=oo=)"; - return (jobType.isProduction() && ! isTested(application, versions) - ? testJobsFor(application, versions, reason, clock.instant()).stream() - : Stream.of(deploymentJob(application, versions, application.change(), jobType, reason, clock.instant()))) - .peek(this::trigger) - .map(Job::jobType).collect(toList()); - } - - /** Returns whether the given job is currently running; false if completed since last triggered, asking the build service otherwise. */ - private boolean isRunning(Application application, JobType jobType) { - return ! application.deploymentJobs().statusOf(jobType) - .flatMap(job -> job.lastCompleted().map(run -> run.at().isAfter(job.lastTriggered().get().at()))) - .orElse(false) - && jobStateIsAmong(application, jobType, running, queued); - } - - private boolean jobStateIsAmong(Application application, JobType jobType, JobState state, JobState... states) { - return EnumSet.of(state, states).contains(buildService.stateOf(BuildJob.of(application.id(), - application.deploymentJobs().projectId().getAsLong(), - jobType.jobName()))); - } - - private Job deploymentJob(Application application, Versions versions, Change change, JobType jobType, String reason, Instant availableSince) { - boolean isRetry = application.deploymentJobs().statusOf(jobType).flatMap(JobStatus::jobError) - .filter(JobError.outOfCapacity::equals).isPresent(); - if (isRetry) reason += "; retrying on out of capacity"; - - JobRun triggering = JobRun.triggering(versions.targetPlatform, versions.targetApplication, versions.sourcePlatform, versions.sourceApplication, reason, clock.instant()); - return new Job(application, triggering, jobType, availableSince, isRetry, change.application().isPresent()); + private ApplicationController applications() { + return controller.applications(); } - private Version targetPlatform(Application application, Change change, Optional<Deployment> deployment) { - return max(deployment.map(Deployment::version), change.platform()) - .orElse(application.oldestDeployedPlatform() - .orElse(controller.systemVersion())); + private Optional<JobRun> successOn(Application application, JobType jobType, Versions versions) { + return application.deploymentJobs().statusOf(jobType).flatMap(JobStatus::lastSuccess) + .filter(run -> targetsMatch(versions, run)); } - private ApplicationVersion targetApplication(Application application, Change change, Optional<Deployment> deployment) { - return max(deployment.map(Deployment::applicationVersion), change.application()) - .orElse(application.oldestDeployedApplication() - .orElse(application.deploymentJobs().jobStatus().get(component).lastSuccess().get().application())); + private Optional<Deployment> deploymentFor(Application application, JobType jobType) { + return Optional.ofNullable(application.deployments().get(jobType.zone(controller.system()).get())); } private static <T extends Comparable<T>> Optional<T> max(Optional<T> o1, Optional<T> o2) { return ! o1.isPresent() ? o2 : ! o2.isPresent() ? o1 : o1.get().compareTo(o2.get()) >= 0 ? o1 : o2; } + // ---------- Ready job computation ---------- + /** Returns the set of all jobs which have changes to propagate from the upstream steps. */ private Stream<Job> computeReadyJobs() { return ApplicationList.from(applications().asList()) @@ -339,79 +317,30 @@ public class DeploymentTrigger { return jobs; } - private Change remainingChange(Application application) { - List<JobType> jobs = (application.deploymentSpec().steps().isEmpty() - ? singletonList(new DeploymentSpec.DeclaredZone(test)) - : application.deploymentSpec().steps()).stream() - .flatMap(step -> step.zones().stream()) - .map(order::toJob) - .collect(toList()); - - boolean platformComplete = application.change().platform().map(Change::of) - .map(change -> jobs.stream().allMatch(job -> completedAt(change, application, job).isPresent())) - .orElse(false); - - boolean applicationComplete = application.change().application().map(Change::of) - .map(change -> jobs.stream().allMatch(job -> completedAt(change, application, job).isPresent())) - .orElse(false); - - Change change = application.change(); - if (platformComplete) change = change.withoutPlatform(); - if (applicationComplete) change = change.withoutApplication(); - return change; - } - - /** - * Returns the list of test jobs that should run now, and that need to succeed on the given versions for it to be considered tested. - */ - private List<Job> testJobsFor(Application application, Versions versions, String reason, Instant availableSince) { - List<Step> steps = application.deploymentSpec().steps(); - if (steps.isEmpty()) steps = singletonList(new DeploymentSpec.DeclaredZone(test)); - List<Job> jobs = new ArrayList<>(); - for (Step step : steps.stream().filter(step -> step.deploysTo(test) || step.deploysTo(staging)).collect(toList())) { - for (JobType jobType : step.zones().stream().map(order::toJob).collect(toList())) { - Optional<JobRun> completion = successOn(application, jobType, versions) - .filter(run -> jobType != stagingTest || sourcesMatchIfPresent(versions, run)); - if ( ! completion.isPresent() && jobStateIsAmong(application, jobType, idle)) - jobs.add(deploymentJob(application, versions, application.change(), jobType, reason, availableSince)); - } - } - return jobs; - } - - private boolean isTested(Application application, Versions versions) { - return testedAt(application, versions).isPresent() - || alreadyTriggered(application, versions); - } + // ---------- Job state helpers ---------- - /** If the given state's sources are present and differ from its targets, returns whether they are equal to those of the given job run. */ - private static boolean sourcesMatchIfPresent(Versions versions, JobRun jobRun) { - return ( ! versions.sourcePlatform.filter(version -> ! version.equals(versions.targetPlatform)).isPresent() - || versions.sourcePlatform.equals(jobRun.sourcePlatform())) - && ( ! versions.sourceApplication.filter(version -> ! version.equals(versions.targetApplication)).isPresent() - || versions.sourceApplication.equals(jobRun.sourceApplication())); + private List<JobType> runningProductionJobsFor(Application application) { + return application.deploymentJobs().jobStatus().keySet().parallelStream() + .filter(job -> job.isProduction()) + .filter(job -> isRunning(application, job)) + .collect(toList()); } - private static boolean targetsMatch(Versions versions, JobRun jobRun) { - return versions.targetPlatform.equals(jobRun.platform()) && versions.targetApplication.equals(jobRun.application()); + /** Returns whether the given job is currently running; false if completed since last triggered, asking the build service otherwise. */ + private boolean isRunning(Application application, JobType jobType) { + return ! application.deploymentJobs().statusOf(jobType) + .flatMap(job -> job.lastCompleted().map(run -> run.at().isAfter(job.lastTriggered().get().at()))) + .orElse(false) + && jobStateIsAmong(application, jobType, running, queued); } - private Optional<Instant> testedAt(Application application, Versions versions) { - Optional<JobRun> testRun = successOn(application, systemTest, versions); - Optional<JobRun> stagingRun = successOn(application, stagingTest, versions) - .filter(run -> sourcesMatchIfPresent(versions, run)); - return max(testRun.map(JobRun::at), stagingRun.map(JobRun::at)) - .filter(__ -> testRun.isPresent() && stagingRun.isPresent()); + private boolean jobStateIsAmong(Application application, JobType jobType, JobState state, JobState... states) { + return EnumSet.of(state, states).contains(buildService.stateOf(BuildJob.of(application.id(), + application.deploymentJobs().projectId().getAsLong(), + jobType.jobName()))); } - private boolean alreadyTriggered(Application application, Versions versions) { - return application.deploymentJobs().jobStatus().values().stream() - .filter(job -> job.type().isProduction()) - .anyMatch(job -> job.lastTriggered() - .filter(run -> targetsMatch(versions, run)) - .filter(run -> sourcesMatchIfPresent(versions, run)) - .isPresent()); - } + // ---------- Completion logic ---------- /** * Returns the instant when the given change is complete for the given application for the given job. @@ -436,30 +365,97 @@ public class DeploymentTrigger { .map(Deployment::at); } - private Optional<JobRun> successOn(Application application, JobType jobType, Versions versions) { - return application.deploymentJobs().statusOf(jobType).flatMap(JobStatus::lastSuccess) - .filter(run -> targetsMatch(versions, run)); + private boolean isTested(Application application, Versions versions) { + return testedAt(application, versions).isPresent() + || alreadyTriggered(application, versions); } - private List<JobType> runningProductionJobsFor(Application application) { - return application.deploymentJobs().jobStatus().keySet().parallelStream() - .filter(job -> job.isProduction()) - .filter(job -> isRunning(application, job)) - .collect(toList()); + private Optional<Instant> testedAt(Application application, Versions versions) { + Optional<JobRun> testRun = successOn(application, systemTest, versions); + Optional<JobRun> stagingRun = successOn(application, stagingTest, versions) + .filter(run -> sourcesMatchIfPresent(versions, run)); + return max(testRun.map(JobRun::at), stagingRun.map(JobRun::at)) + .filter(__ -> testRun.isPresent() && stagingRun.isPresent()); } - private ApplicationController applications() { - return controller.applications(); + private boolean alreadyTriggered(Application application, Versions versions) { + return application.deploymentJobs().jobStatus().values().stream() + .filter(job -> job.type().isProduction()) + .anyMatch(job -> job.lastTriggered() + .filter(run -> targetsMatch(versions, run)) + .filter(run -> sourcesMatchIfPresent(versions, run)) + .isPresent()); } + /** If the given state's sources are present and differ from its targets, returns whether they are equal to those of the given job run. */ + private static boolean sourcesMatchIfPresent(Versions versions, JobRun jobRun) { + return ( ! versions.sourcePlatform.filter(version -> ! version.equals(versions.targetPlatform)).isPresent() + || versions.sourcePlatform.equals(jobRun.sourcePlatform())) + && ( ! versions.sourceApplication.filter(version -> ! version.equals(versions.targetApplication)).isPresent() + || versions.sourceApplication.equals(jobRun.sourceApplication())); + } + + private static boolean targetsMatch(Versions versions, JobRun jobRun) { + return versions.targetPlatform.equals(jobRun.platform()) && versions.targetApplication.equals(jobRun.application()); + } + + // ---------- Change management o_O ---------- + private boolean acceptNewApplicationVersion(Application application) { if (application.change().application().isPresent()) return true; // More application changes are ok. if (application.deploymentJobs().hasFailures()) return true; // Allow changes to fix upgrade problems. return ! application.changeAt(clock.instant()).platform().isPresent(); } - private Optional<Deployment> deploymentFor(Application application, JobType jobType) { - return Optional.ofNullable(application.deployments().get(jobType.zone(controller.system()).get())); + private Change remainingChange(Application application) { + List<JobType> jobs = (application.deploymentSpec().steps().isEmpty() + ? singletonList(new DeploymentSpec.DeclaredZone(test)) + : application.deploymentSpec().steps()).stream() + .flatMap(step -> step.zones().stream()) + .map(order::toJob) + .collect(toList()); + + boolean platformComplete = application.change().platform().map(Change::of) + .map(change -> jobs.stream().allMatch(job -> completedAt(change, application, job).isPresent())) + .orElse(false); + + boolean applicationComplete = application.change().application().map(Change::of) + .map(change -> jobs.stream().allMatch(job -> completedAt(change, application, job).isPresent())) + .orElse(false); + + Change change = application.change(); + if (platformComplete) change = change.withoutPlatform(); + if (applicationComplete) change = change.withoutApplication(); + return change; + } + + // ---------- Version and job helpers ---------- + + /** + * Returns the list of test jobs that should run now, and that need to succeed on the given versions for it to be considered tested. + */ + private List<Job> testJobsFor(Application application, Versions versions, String reason, Instant availableSince) { + List<Step> steps = application.deploymentSpec().steps(); + if (steps.isEmpty()) steps = singletonList(new DeploymentSpec.DeclaredZone(test)); + List<Job> jobs = new ArrayList<>(); + for (Step step : steps.stream().filter(step -> step.deploysTo(test) || step.deploysTo(staging)).collect(toList())) { + for (JobType jobType : step.zones().stream().map(order::toJob).collect(toList())) { + Optional<JobRun> completion = successOn(application, jobType, versions) + .filter(run -> jobType != stagingTest || sourcesMatchIfPresent(versions, run)); + if ( ! completion.isPresent() && jobStateIsAmong(application, jobType, idle)) + jobs.add(deploymentJob(application, versions, application.change(), jobType, reason, availableSince)); + } + } + return jobs; + } + + private Job deploymentJob(Application application, Versions versions, Change change, JobType jobType, String reason, Instant availableSince) { + boolean isRetry = application.deploymentJobs().statusOf(jobType).flatMap(JobStatus::jobError) + .filter(JobError.outOfCapacity::equals).isPresent(); + if (isRetry) reason += "; retrying on out of capacity"; + + JobRun triggering = JobRun.triggering(versions.targetPlatform, versions.targetApplication, versions.sourcePlatform, versions.sourceApplication, reason, clock.instant()); + return new Job(application, triggering, jobType, availableSince, isRetry, change.application().isPresent()); } private Versions versionsFor(Application application, Change change, Optional<Deployment> deployment) { @@ -469,6 +465,20 @@ public class DeploymentTrigger { deployment.map(Deployment::applicationVersion)); } + private Version targetPlatform(Application application, Change change, Optional<Deployment> deployment) { + return max(deployment.map(Deployment::version), change.platform()) + .orElse(application.oldestDeployedPlatform() + .orElse(controller.systemVersion())); + } + + private ApplicationVersion targetApplication(Application application, Change change, Optional<Deployment> deployment) { + return max(deployment.map(Deployment::applicationVersion), change.application()) + .orElse(application.oldestDeployedApplication() + .orElse(application.deploymentJobs().jobStatus().get(component).lastSuccess().get().application())); + } + + // ---------- Data containers ---------- + private static class Job extends BuildJob { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index a7c600c4ae4..93103213274 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -50,7 +50,7 @@ public class ControllerMaintenance extends AbstractComponent { deploymentIssueReporter = new DeploymentIssueReporter(controller, deploymentIssues, maintenanceInterval, jobControl); metricsReporter = new MetricsReporter(controller, metric, chefClient, jobControl, controller.system()); outstandingChangeDeployer = new OutstandingChangeDeployer(controller, maintenanceInterval, jobControl); - versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(3), jobControl); + versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(1), jobControl); upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator); readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofSeconds(30), jobControl); clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl, nodeRepositoryClient); diff --git a/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java index fe9611b0a37..3e6d95c6c54 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/SendProxyTestCase.java @@ -13,7 +13,6 @@ import com.yahoo.messagebus.test.Receptor; import com.yahoo.messagebus.test.SimpleMessage; import com.yahoo.messagebus.test.SimpleProtocol; import com.yahoo.messagebus.test.SimpleReply; -import junit.framework.TestCase; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/messagebus/src/test/java/com/yahoo/messagebus/routing/RetryPolicyTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/routing/RetryPolicyTestCase.java index a78b667a530..41cd0897f9e 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/routing/RetryPolicyTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/routing/RetryPolicyTestCase.java @@ -2,13 +2,18 @@ package com.yahoo.messagebus.routing; import com.yahoo.messagebus.ErrorCode; -import junit.framework.TestCase; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ -public class RetryPolicyTestCase extends TestCase { +public class RetryPolicyTestCase { + @Test public void testSimpleRetryPolicy() { RetryTransientErrorsPolicy policy = new RetryTransientErrorsPolicy(); for (int i = 0; i < 5; ++i) { @@ -29,4 +34,5 @@ public class RetryPolicyTestCase extends TestCase { } } } + } diff --git a/messagebus/src/test/java/com/yahoo/messagebus/routing/RouteParserTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/routing/RouteParserTestCase.java index fde72ded150..c2e4aee1bd8 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/routing/RouteParserTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/routing/RouteParserTestCase.java @@ -1,11 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.routing; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ -public class RouteParserTestCase extends junit.framework.TestCase { +public class RouteParserTestCase { + @Test public void testHopParser() { Hop hop = Hop.parse("foo"); assertNotNull(hop); @@ -64,6 +71,7 @@ public class RouteParserTestCase extends junit.framework.TestCase { "route[0].feed \"myfeed\""); } + @Test public void testHopParserErrors() { assertError(Hop.parse(""), "Failed to parse empty string."); assertError(Hop.parse("[foo"), "Unterminated '[' in '[foo'"); @@ -71,6 +79,7 @@ public class RouteParserTestCase extends junit.framework.TestCase { assertError(Hop.parse("foo bar"), "Failed to completely parse 'foo bar'."); } + @Test public void testShortRoute() { Route shortRoute = Route.parse("c"); assertNotNull(shortRoute); @@ -81,6 +90,7 @@ public class RouteParserTestCase extends junit.framework.TestCase { assertVerbatimDirective(hop.getDirective(0), "c"); } + @Test public void testShortHops() { Route shortRoute = Route.parse("a b c"); assertNotNull(shortRoute); @@ -91,6 +101,7 @@ public class RouteParserTestCase extends junit.framework.TestCase { assertVerbatimDirective(hop.getDirective(0), "a"); } + @Test public void testRouteParser() { Route route = Route.parse("foo bar/baz"); assertNotNull(route); @@ -114,6 +125,7 @@ public class RouteParserTestCase extends junit.framework.TestCase { assertVerbatimDirective(hop.getDirective(0), "default"); } + @Test public void testRouteParserErrors() { assertError(Route.parse(""), "Failed to parse empty string."); assertError(Route.parse("foo [bar"), "Unterminated '[' in '[bar'"); @@ -165,4 +177,5 @@ public class RouteParserTestCase extends junit.framework.TestCase { assertTrue(dir instanceof VerbatimDirective); assertEquals(image, ((VerbatimDirective)dir).getImage()); } + } diff --git a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingContextTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingContextTestCase.java index 0c738db4242..527896e2b44 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingContextTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingContextTestCase.java @@ -11,30 +11,27 @@ import com.yahoo.messagebus.network.rpc.test.TestServer; import com.yahoo.messagebus.test.Receptor; import com.yahoo.messagebus.test.SimpleMessage; import com.yahoo.messagebus.test.SimpleProtocol; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; -import java.net.UnknownHostException; import java.util.Arrays; import java.util.List; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ public class RoutingContextTestCase extends junit.framework.TestCase { - public static final int TIMEOUT_SECS = 120; - //////////////////////////////////////////////////////////////////////////////// - // - // Setup - // - //////////////////////////////////////////////////////////////////////////////// + public static final int TIMEOUT_SECS = 120; Slobrok slobrok; TestServer srcServer, dstServer; SourceSession srcSession; DestinationSession dstSession; - @Override - public void setUp() throws ListenFailedException, UnknownHostException { + @Before + public void setUp() throws ListenFailedException { slobrok = new Slobrok(); dstServer = new TestServer(new MessageBusParams().addProtocol(new SimpleProtocol()), new RPCNetworkParams().setIdentity(new Identity("dst")).setSlobrokConfigId(TestServer.getSlobrokConfig(slobrok))); @@ -46,7 +43,7 @@ public class RoutingContextTestCase extends junit.framework.TestCase { assertTrue(srcServer.waitSlobrok("dst/session", 1)); } - @Override + @After public void tearDown() { slobrok.stop(); dstSession.destroy(); @@ -55,12 +52,7 @@ public class RoutingContextTestCase extends junit.framework.TestCase { srcServer.destroy(); } - //////////////////////////////////////////////////////////////////////////////// - // - // Tests - // - //////////////////////////////////////////////////////////////////////////////// - + @Test public void testSingleDirective() { SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory( @@ -81,6 +73,7 @@ public class RoutingContextTestCase extends junit.framework.TestCase { } } + @Test public void testMoreDirectives() { SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("Custom", new CustomPolicyFactory( @@ -103,6 +96,7 @@ public class RoutingContextTestCase extends junit.framework.TestCase { } } + @Test public void testRecipientsRemain() { SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("First", new CustomPolicyFactory(true, Arrays.asList("foo/bar"), Arrays.asList("foo/[Second]"))); @@ -121,10 +115,12 @@ public class RoutingContextTestCase extends junit.framework.TestCase { } } + @Test public void testToString() { assertEquals("node : null, directive: 1, errors: [], selectOnRetry: true context: null", new RoutingContext(null, 1).toString()); } + @Test public void testConstRoute() { SimpleProtocol protocol = new SimpleProtocol(); protocol.addPolicyFactory("DocumentRouteSelector", @@ -146,12 +142,6 @@ public class RoutingContextTestCase extends junit.framework.TestCase { } } - //////////////////////////////////////////////////////////////////////////////// - // - // Utilities - // - //////////////////////////////////////////////////////////////////////////////// - private Message createMessage(String msg) { Message ret = new SimpleMessage(msg); ret.getTrace().setLevel(9); diff --git a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingSpecTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingSpecTestCase.java index 8ebbe664d8d..e7ea6346999 100755 --- a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingSpecTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingSpecTestCase.java @@ -3,24 +3,23 @@ package com.yahoo.messagebus.routing; import com.yahoo.messagebus.ConfigAgent; import com.yahoo.messagebus.ConfigHandler; -import junit.framework.TestCase; +import org.junit.Test; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ -public class RoutingSpecTestCase extends TestCase { - - //////////////////////////////////////////////////////////////////////////////// - // - // Tests - // - //////////////////////////////////////////////////////////////////////////////// +public class RoutingSpecTestCase { + @Test public void testConfig() { assertConfig(new RoutingSpec()); assertConfig(new RoutingSpec().addTable(new RoutingTableSpec("mytable1"))); @@ -71,6 +70,7 @@ public class RoutingSpecTestCase extends TestCase { .addRoute(new RouteSpec("myroute1").addHop("myhop1"))).toString()); } + @Test public void testApplicationSpec() { assertApplicationSpec(Arrays.asList("foo"), Arrays.asList("foo", @@ -101,6 +101,7 @@ public class RoutingSpecTestCase extends TestCase { "*/*/*")); } + @Test public void testVeriyfOk() { assertVerifyOk(new RoutingSpec().addTable(new RoutingTableSpec("mytable") .addHop(new HopSpec("hop1", "myservice1"))), @@ -125,6 +126,7 @@ public class RoutingSpecTestCase extends TestCase { .addService("mytable", "foo/1/baz")); } + @Test public void testVerifyToggle() { assertVerifyOk(new RoutingSpec(false) .addTable(new RoutingTableSpec("mytable")) @@ -142,6 +144,7 @@ public class RoutingSpecTestCase extends TestCase { new ApplicationSpec()); } + @Test public void testVerifyFail() { // Duplicate table. assertVerifyFail(new RoutingSpec() @@ -274,12 +277,6 @@ public class RoutingSpecTestCase extends TestCase { "Selector 'bar/[baz]/cox' does not match recipient 'cox/0/bar' in hop 'hop5' in routing table 'mytable'.")); } - //////////////////////////////////////////////////////////////////////////////// - // - // Utilities - // - //////////////////////////////////////////////////////////////////////////////// - private static void assertVerifyOk(RoutingSpec routing, ApplicationSpec app) { assertVerifyFail(routing, app, new ArrayList<String>()); } @@ -333,4 +330,5 @@ public class RoutingSpecTestCase extends TestCase { this.routing = routing; } } + } diff --git a/metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java b/metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java index c066814b557..ef802a73c4c 100644 --- a/metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java +++ b/metrics/src/test/java/com/yahoo/metrics/CountMetricTest.java @@ -1,13 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.metrics; -import junit.framework.TestCase; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; /** * @author thomasg */ -public class CountMetricTest extends TestCase { +public class CountMetricTest { + @Test public void testCountMetric() { CountMetric m = new CountMetric("test", "tag", "description", null); assertEquals(false, m.used()); @@ -36,7 +39,7 @@ public class CountMetricTest extends TestCase { assertEquals("<test description=\"description\" count=\"96\"/>\n", m2.toXml(0, 2)); assertEquals("<test description=\"description\" count=\"96\" average_change_per_second=\"9.60\"/>\n", m2.toXml(10, 2)); - assertEquals(96.0, m2.getDoubleValue("value")); + assertEquals(96.0, m2.getDoubleValue("value"), 0.00000001); assertEquals(96, m2.getLongValue("value")); } diff --git a/metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java b/metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java index 177a2bbb993..bfe10825f4f 100644 --- a/metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java +++ b/metrics/src/test/java/com/yahoo/metrics/MetricSetTest.java @@ -1,10 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.metrics; -import junit.framework.TestCase; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; -public class MetricSetTest extends TestCase { +public class MetricSetTest { + + private final double delta = 0.000000001; class TestMetricVisitor extends MetricVisitor { String output = ""; @@ -28,8 +33,8 @@ public class MetricSetTest extends TestCase { } } - public void testNormalUsage() - { + @Test + public void testNormalUsage() { // Set up some metrics to test.. MetricSet set = new SimpleMetricSet("a", "foo", ""); SummedDoubleValueMetric v1 = new SummedDoubleValueMetric("c", "foo", "", set); @@ -111,6 +116,7 @@ public class MetricSetTest extends TestCase { } @SuppressWarnings("rawtypes") + @Test public void testSumInSet() { MetricSet ms_a = new SimpleMetricSet("a", "", "", null); MyTimer timer = new MyTimer(); @@ -151,11 +157,11 @@ public class MetricSetTest extends TestCase { ValueMetric av_2 = (ValueMetric)sms.getMetric("v2"); ValueMetric av_3 = (ValueMetric)sms.getMetric("v3"); - assertEquals(10.0, av_sum.getDoubleValue("total")); - assertEquals(4, av_sum.getLongValue("count")); - assertEquals(4.0, av_1.getDoubleValue("total")); - assertEquals(4.0, av_2.getDoubleValue("total")); - assertEquals(2.0, av_3.getDoubleValue("total")); + assertEquals(10.0, av_sum.getDoubleValue("total"), delta); + assertEquals(4, av_sum.getLongValue("count"), delta); + assertEquals(4.0, av_1.getDoubleValue("total"), delta); + assertEquals(4.0, av_2.getDoubleValue("total"), delta); + assertEquals(2.0, av_3.getDoubleValue("total"), delta); timer.timeNow = 301; manager.takeSnapshots(301); diff --git a/metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java b/metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java index eda3c532c19..008575af574 100644 --- a/metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java +++ b/metrics/src/test/java/com/yahoo/metrics/SumMetricTest.java @@ -1,12 +1,18 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.metrics; -import junit.framework.TestCase; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; /** * @author thomasg */ -public class SumMetricTest extends TestCase { +public class SumMetricTest { + + private final double delta = 0.000000001; + + @Test public void testSummedCountMetric() { MetricSet parent = new SimpleMetricSet("parent", "", ""); SumMetric sum = new SumMetric("foo", "", "foodesc", parent); @@ -33,6 +39,7 @@ public class SumMetricTest extends TestCase { assertEquals(20, sum.getLongValue("value")); } + @Test public void testSummedValueMetric() { MetricSet parent = new SimpleMetricSet("parent", "", ""); SumMetric sum = new SumMetric("foo", "", "foodesc", parent); @@ -60,7 +67,7 @@ public class SumMetricTest extends TestCase { assertEquals(10, sum.getLongValue("max")); } - + @Test public void testAveragedValueMetric() { MetricSet parent = new SimpleMetricSet("parent", "", ""); SumMetric sum = new SumMetric("foo", "", "foodesc", parent); @@ -83,11 +90,12 @@ public class SumMetricTest extends TestCase { assertEquals("<foo description=\"foodesc\" average=\"5.40\" last=\"10.00\" min=\"2.00\" max=\"10.00\" count=\"5\" total=\"27.00\"/>\n", sum.toXml(0,2)); - assertEquals(5.40, sum.getDoubleValue("value")); - assertEquals(2.0, sum.getDoubleValue("min")); - assertEquals(10.0, sum.getDoubleValue("max")); + assertEquals(5.40, sum.getDoubleValue("value"), delta); + assertEquals(2.0, sum.getDoubleValue("min"), delta); + assertEquals(10.0, sum.getDoubleValue("max"), delta); } + @Test public void testMetricSet() { MetricSet parent = new SimpleMetricSet("parent", "", ""); SumMetric sum = new SumMetric("foo", "", "bar", parent); @@ -118,9 +126,8 @@ public class SumMetricTest extends TestCase { } - - public void testRemove() - { + @Test + public void testRemove() { MetricSet parent = new SimpleMetricSet("parent", "", ""); SumMetric sum = new SumMetric("foo", "", "foodesc", parent); @@ -145,6 +152,7 @@ public class SumMetricTest extends TestCase { assertEquals("<foo description=\"foodesc\" average=\"14.50\" last=\"17.00\" min=\"5.00\" max=\"10.00\" count=\"3\" total=\"43.50\"/>\n", sum.toXml(0,2)); } + @Test public void testEmpty() { SumMetric sum = new SumMetric("foo", "", "foodesc", null); assertEquals("<foo description=\"foodesc\"/>\n", sum.toXml(0,2)); diff --git a/metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java b/metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java index 1054dacdcbb..40ba70a9749 100644 --- a/metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java +++ b/metrics/src/test/java/com/yahoo/metrics/ValueMetricTest.java @@ -1,11 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.metrics; -import junit.framework.TestCase; +import org.junit.Test; -public class ValueMetricTest extends TestCase { - public void testAveragedDoubleValueMetric() - { +import static org.junit.Assert.assertEquals; + +public class ValueMetricTest { + + private static final double delta = 0.000000001; + + @Test + public void testAveragedDoubleValueMetric() { AveragedDoubleValueMetric m = new AveragedDoubleValueMetric("test", "tag", "description", null); m.addValue(100.0); assertEquals("count=\"1\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"100.00\" average=\"100.00\"", m.toString()); @@ -43,13 +48,13 @@ public class ValueMetricTest extends TestCase { assertEquals("<test description=\"description\" average=\"343.33\" last=\"2000.00\" min=\"40.00\" max=\"2000.00\" count=\"9\" total=\"3090.00\"/>\n", sum.toXml(0, 2)); - assertEquals(80.0, m2.getDoubleValue("value")); - assertEquals(80.0, m2.getDoubleValue("average")); - assertEquals(40.0, m2.getDoubleValue("min")); - assertEquals(100.0, m2.getDoubleValue("max")); - assertEquals(40.0, m2.getDoubleValue("last")); - assertEquals(3.0, m2.getDoubleValue("count")); - assertEquals(240.0, m2.getDoubleValue("total")); + assertEquals(80.0, m2.getDoubleValue("value"), delta); + assertEquals(80.0, m2.getDoubleValue("average"), delta); + assertEquals(40.0, m2.getDoubleValue("min"), delta); + assertEquals(100.0, m2.getDoubleValue("max"), delta); + assertEquals(40.0, m2.getDoubleValue("last"), delta); + assertEquals(3.0, m2.getDoubleValue("count"), delta); + assertEquals(240.0, m2.getDoubleValue("total"), delta); assertEquals(80, m2.getLongValue("value")); assertEquals(80, m2.getLongValue("average")); @@ -60,20 +65,22 @@ public class ValueMetricTest extends TestCase { assertEquals(240, m2.getLongValue("total")); } + @Test public void testDoubleValueMetricNotUpdatedOnNaN() { AveragedDoubleValueMetric m = new AveragedDoubleValueMetric("test", "tag", "description", null); m.addValue(Double.NaN); assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString()); } + @Test public void testDoubleValueMetricNotUpdatedOnInfinity() { AveragedDoubleValueMetric m = new AveragedDoubleValueMetric("test", "tag", "description", null); m.addValue(Double.POSITIVE_INFINITY); assertEquals("count=\"0\" min=\"0\" max=\"0\" last=\"0\" total=\"0\" average=\"0.00\"", m.toString()); } - public void testSummedDoubleValueMetric() - { + @Test + public void testSummedDoubleValueMetric() { SummedDoubleValueMetric m = new SummedDoubleValueMetric("test", "tag", "description", null); m.addValue(100.0); assertEquals("count=\"1\" min=\"100.00\" max=\"100.00\" last=\"100.00\" total=\"100.00\" average=\"100.00\"", m.toString()); @@ -109,13 +116,13 @@ public class ValueMetricTest extends TestCase { assertEquals("<test description=\"description\" average=\"2313.33\" last=\"2440.00\" min=\"40.00\" max=\"2000.00\" count=\"7\" total=\"16193.33\"/>\n", sum.toXml(0, 2)); - assertEquals(40.0, m2.getDoubleValue("value")); - assertEquals(80.0, m2.getDoubleValue("average")); - assertEquals(40.0, m2.getDoubleValue("min")); - assertEquals(100.0, m2.getDoubleValue("max")); - assertEquals(40.0, m2.getDoubleValue("last")); - assertEquals(3.0, m2.getDoubleValue("count")); - assertEquals(240.0, m2.getDoubleValue("total")); + assertEquals(40.0, m2.getDoubleValue("value"), delta); + assertEquals(80.0, m2.getDoubleValue("average"), delta); + assertEquals(40.0, m2.getDoubleValue("min"), delta); + assertEquals(100.0, m2.getDoubleValue("max"), delta); + assertEquals(40.0, m2.getDoubleValue("last"), delta); + assertEquals(3.0, m2.getDoubleValue("count"), delta); + assertEquals(240.0, m2.getDoubleValue("total"), delta); assertEquals(40, m2.getLongValue("value")); assertEquals(80, m2.getLongValue("average")); @@ -126,12 +133,12 @@ public class ValueMetricTest extends TestCase { assertEquals(240, m2.getLongValue("total")); } - public void testAveragedLongValueMetric() - { + @Test + public void testAveragedLongValueMetric() { AveragedLongValueMetric m = new AveragedLongValueMetric("test", "tag", "description", null); - assertEquals(0l, m.getLongValue("max")); - assertEquals(0l, m.getLongValue("min")); + assertEquals(0L, m.getLongValue("max")); + assertEquals(0L, m.getLongValue("min")); m.addValue((long)100); assertEquals("count=\"1\" min=\"100\" max=\"100\" last=\"100\" total=\"100\" average=\"100.00\"", m.toString()); @@ -178,5 +185,4 @@ public class ValueMetricTest extends TestCase { assertEquals(240, m2.getLongValue("total")); } - } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java index 218b2947a21..5dd80961062 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; import java.net.URI; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -28,6 +29,7 @@ public class ConfigServerInfo { private final Optional<KeyStoreOptions> keyStoreOptions; private final Optional<KeyStoreOptions> trustStoreOptions; private final Optional<AthenzIdentity> athenzIdentity; + private final Optional<ConfigServerConfig.Sia> siaConfig; public ConfigServerInfo(ConfigServerConfig config) { this.configServerHostNames = config.hosts(); @@ -46,6 +48,7 @@ public class ConfigServerInfo { this.athenzIdentity = createAthenzIdentity( config.athenzDomain(), config.serviceName()); + this.siaConfig = verifySiaConfig(config.sia()); } public List<String> getConfigServerHostNames() { @@ -77,6 +80,10 @@ public class ConfigServerInfo { return athenzIdentity; } + public Optional<ConfigServerConfig.Sia> getSiaConfig() { + return siaConfig; + } + private static Map<String, URI> createConfigServerUris( String scheme, List<String> configServerHosts, @@ -86,6 +93,18 @@ public class ConfigServerInfo { hostname -> URI.create(scheme + "://" + hostname + ":" + port))); } + private static Optional<ConfigServerConfig.Sia> verifySiaConfig(ConfigServerConfig.Sia sia) { + List<String> configParams = Arrays.asList( + sia.credentialsPath(), sia.configserverIdentityName(), sia.hostIdentityName(), sia.trustStoreFile()); + if (configParams.stream().allMatch(String::isEmpty)) { + return Optional.empty(); + } else if (configParams.stream().noneMatch(String::isEmpty)) { + return Optional.of(sia); + } else { + throw new IllegalArgumentException("Inconsistent sia config: " + sia); + } + } + private static Optional<KeyStoreOptions> createKeyStoreOptions(String pathToKeyStore, char[] password, String type) { return Optional.ofNullable(pathToKeyStore) .filter(path -> !Strings.isNullOrEmpty(path)) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index fd34fede291..c29028c493e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -109,12 +109,11 @@ public class ConfigServerApiImpl implements ConfigServerApi { private ConfigServerApiImpl(Collection<URI> configServerUris, SSLConnectionSocketFactory sslConnectionSocketFactory) { - this(randomizeConfigServerUris(configServerUris), - new SelfCloseableHttpClient(sslConnectionSocketFactory)); + this(configServerUris, new SelfCloseableHttpClient(sslConnectionSocketFactory)); } - private ConfigServerApiImpl(List<URI> configServerHosts, SelfCloseableHttpClient client) { - this.configServerHosts = configServerHosts; + private ConfigServerApiImpl(Collection<URI> configServerHosts, SelfCloseableHttpClient client) { + this.configServerHosts = randomizeConfigServerUris(configServerHosts); this.client = client; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java index 1405a518625..13af642af4a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/RealConfigServerClients.java @@ -29,7 +29,7 @@ public class RealConfigServerClients implements ConfigServerClients { private final ConfigServerInfo configServerInfo; public RealConfigServerClients(Environment environment) { - this(environment.getConfigServerInfo(), environment.getParentHostHostname()); + this(environment.getConfigServerInfo()); } /** @@ -39,9 +39,9 @@ public class RealConfigServerClients implements ConfigServerClients { * and kept up to date. On failure, this constructor will throw an exception and * the caller may retry later. */ - public RealConfigServerClients(ConfigServerInfo info, String hostname) { + public RealConfigServerClients(ConfigServerInfo info) { this.configServerInfo = info; - updater = SslConnectionSocketFactoryUpdater.createAndRefreshKeyStoreIfNeeded(info, hostname); + updater = SslConnectionSocketFactoryUpdater.createAndRefreshKeyStoreIfNeeded(info); configServerApi = ConfigServerApiImpl.create(info, updater); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryCreator.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryCreator.java deleted file mode 100644 index 85b2b426954..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryCreator.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.configserver; - -import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; -import com.yahoo.vespa.athenz.tls.SslContextBuilder; -import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; -import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLContext; -import java.util.Collections; -import java.util.Optional; - -/** - * @author hakon - */ -class SslConnectionSocketFactoryCreator { - SSLConnectionSocketFactory createSocketFactory( - ConfigServerInfo configServerInfo, - Optional<KeyStoreOptions> keyStoreOptions) { - SSLContext context = makeSslContext(configServerInfo, keyStoreOptions); - return new SSLConnectionSocketFactory(context, makeHostnameVerifier(configServerInfo)); - } - - private static SSLContext makeSslContext( - ConfigServerInfo configServerInfo, - Optional<KeyStoreOptions> keyStoreOptions) { - SslContextBuilder sslContextBuilder = new SslContextBuilder(); - configServerInfo.getTrustStoreOptions() - .map(KeyStoreOptions::loadKeyStore) - .ifPresent(sslContextBuilder::withTrustStore); - keyStoreOptions.ifPresent(options -> - sslContextBuilder.withKeyStore(options.loadKeyStore(), options.password)); - - return sslContextBuilder.build(); - } - - private static HostnameVerifier makeHostnameVerifier(ConfigServerInfo configServerInfo) { - return configServerInfo.getAthenzIdentity() - .map(identity -> (HostnameVerifier) new AthenzIdentityVerifier(Collections.singleton(identity))) - .orElseGet(SSLConnectionSocketFactory::getDefaultHostnameVerifier); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdater.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdater.java index 57fa5526d73..5a97174762c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdater.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdater.java @@ -1,91 +1,86 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.identity.SiaIdentityProvider; +import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; +import com.yahoo.vespa.athenz.tls.SslContextBuilder; +import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; -import com.yahoo.vespa.hosted.node.admin.configserver.certificate.ConfigServerKeyStoreRefresher; -import com.yahoo.vespa.hosted.node.admin.configserver.certificate.ConfigServerKeyStoreRefresherFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; +import java.io.File; +import java.nio.file.Paths; import java.util.HashSet; -import java.util.Optional; import java.util.Set; +import static java.util.Collections.singleton; + /** - * Responsible for updating SSLConnectionSocketFactory on ConfigServerApiImpl asynchronously - * and as required by embedded certificate expiry + * Responsible for updating {@link SSLConnectionSocketFactory} on {@link ConfigServerApiImpl} asynchronously + * using SIA based certificates (through {@link SiaIdentityProvider}). * + * @author bjorncs * @author hakon */ public class SslConnectionSocketFactoryUpdater implements AutoCloseable { - private final ConfigServerInfo configServerInfo; - private final SslConnectionSocketFactoryCreator socketFactoryCreator; - // Internal ConfigServerApi used to refresh the key store - private final ConfigServerApiImpl configServerApi; - private final Optional<ConfigServerKeyStoreRefresher> keyStoreRefresher; private final Object monitor = new Object(); - private SSLConnectionSocketFactory socketFactory = null; + private final HostnameVerifier configServerHostnameVerifier; + private final SiaIdentityProvider sia; + private final Set<ConfigServerApi> configServerApis = new HashSet<>(); + private SSLConnectionSocketFactory socketFactory; /** * Creates an updater with valid initial {@link SSLConnectionSocketFactory} * - * @param hostname the hostname of localhost * @throws RuntimeException if e.g. key store options have been specified, but was unable * create a create a key store with a valid certificate */ - public static SslConnectionSocketFactoryUpdater createAndRefreshKeyStoreIfNeeded( - ConfigServerInfo configServerInfo, String hostname) { - return new SslConnectionSocketFactoryUpdater( - configServerInfo, - hostname, - ConfigServerKeyStoreRefresher::new, - new SslConnectionSocketFactoryCreator()); + public static SslConnectionSocketFactoryUpdater createAndRefreshKeyStoreIfNeeded(ConfigServerInfo configServerInfo) { + SiaIdentityProvider siaIdentityProvider = configServerInfo.getSiaConfig() + .map(siaConfig -> + new SiaIdentityProvider( + (AthenzService) AthenzIdentities.from(siaConfig.hostIdentityName()), + Paths.get(siaConfig.credentialsPath()), + new File(siaConfig.trustStoreFile()))) + .orElse(null); + HostnameVerifier configServerHostnameVerifier = configServerInfo.getSiaConfig() + .map(siaConfig -> createHostnameVerifier(AthenzIdentities.from(siaConfig.configserverIdentityName()))) + .orElseGet(SSLConnectionSocketFactory::getDefaultHostnameVerifier); + return new SslConnectionSocketFactoryUpdater(siaIdentityProvider, configServerHostnameVerifier); } - /** Non-private for testing only */ - SslConnectionSocketFactoryUpdater( - ConfigServerInfo configServerInfo, - String hostname, - ConfigServerKeyStoreRefresherFactory refresherFactory, - SslConnectionSocketFactoryCreator socketFactoryCreator) { - this.configServerInfo = configServerInfo; - this.socketFactoryCreator = socketFactoryCreator; - - // ConfigServerApi used to refresh the key store. Does not itself rely on a socket - // factory with key store, of course. - SSLConnectionSocketFactory socketFactoryWithoutKeyStore = - socketFactoryCreator.createSocketFactory(configServerInfo, Optional.empty()); - configServerApi = ConfigServerApiImpl.createWithSocketFactory( - configServerInfo.getConfigServerUris(), socketFactoryWithoutKeyStore); - - // If we have keystore options, we should make sure we use the keystore with the latest certificate, - // start the keystore refresher. - keyStoreRefresher = configServerInfo.getKeyStoreOptions().map(keyStoreOptions -> { - ConfigServerKeyStoreRefresher keyStoreRefresher = refresherFactory.create( - keyStoreOptions, - this::updateSslConnectionSocketFactory, - configServerApi, - hostname); - - // Run the refresh once manually to make sure that we have a valid certificate, otherwise fail. - try { - keyStoreRefresher.refreshKeyStoreIfNeeded(); - updateSslConnectionSocketFactory(); - } catch (Exception e) { - throw new RuntimeException("Failed to acquire certificate to config server", e); - } - - keyStoreRefresher.start(); - return keyStoreRefresher; - }); + SslConnectionSocketFactoryUpdater(SiaIdentityProvider siaIdentityProvider, + HostnameVerifier configServerHostnameVerifier) { + this.configServerHostnameVerifier = configServerHostnameVerifier; + this.sia = siaIdentityProvider; + if (siaIdentityProvider != null) { + siaIdentityProvider.addReloadListener(this::updateSocketFactory); + socketFactory = createSocketFactory(siaIdentityProvider.getIdentitySslContext()); + } else { + socketFactory = createDefaultSslConnectionSocketFactory(); + } + } + + private void updateSocketFactory(SSLContext sslContext) { + synchronized (monitor) { + socketFactory = createSocketFactory(sslContext); + configServerApis.forEach(api -> api.setSSLConnectionSocketFactory(socketFactory)); + } } public SSLConnectionSocketFactory getCurrentSocketFactory() { - return socketFactory; + synchronized (monitor) { + return socketFactory; + } } - /** Register a {@link ConfigServerApi} whose SSLConnectionSocketFactory will be kept up to date */ + /** Register a {@link ConfigServerApi} whose {@link SSLConnectionSocketFactory} will be kept up to date */ public void registerConfigServerApi(ConfigServerApi configServerApi) { synchronized (monitor) { configServerApi.setSSLConnectionSocketFactory(socketFactory); @@ -101,17 +96,22 @@ public class SslConnectionSocketFactoryUpdater implements AutoCloseable { @Override public void close() { - keyStoreRefresher.ifPresent(ConfigServerKeyStoreRefresher::stop); - configServerApi.close(); + if (sia != null) { + sia.deconstruct(); + } } - private void updateSslConnectionSocketFactory() { - synchronized (monitor) { - socketFactory = socketFactoryCreator.createSocketFactory( - configServerInfo, - configServerInfo.getKeyStoreOptions()); + private SSLConnectionSocketFactory createSocketFactory(SSLContext sslContext) { + return new SSLConnectionSocketFactory(sslContext, configServerHostnameVerifier); + } - configServerApis.forEach(api -> api.setSSLConnectionSocketFactory(socketFactory)); - } + private SSLConnectionSocketFactory createDefaultSslConnectionSocketFactory() { + SSLContext sslContext = new SslContextBuilder().build(); + return createSocketFactory(sslContext); + } + + private static HostnameVerifier createHostnameVerifier(AthenzIdentity identity) { + return new AthenzIdentityVerifier(singleton(identity)); } + } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java index ef49822e825..3b47f99ba77 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/Acl.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; import java.net.InetAddress; import java.util.Collections; +import java.util.LinkedList; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -32,34 +33,39 @@ public class Acl { this.trustedPorts = trustedPorts != null ? ImmutableList.copyOf(trustedPorts) : Collections.emptyList(); } - public String toRules(IPVersion ipVersion) { + public List<String> toRules(IPVersion ipVersion) { + List<String> rules = new LinkedList<>(); - String basics = String.join("\n" - // We reject with rules instead of using policies - , "-P INPUT ACCEPT" - , "-P FORWARD ACCEPT" - , "-P OUTPUT ACCEPT" - // Allow packets belonging to established connections - , "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT" - // Allow any loopback traffic - , "-A INPUT -i lo -j ACCEPT" - // Allow ICMP packets. See http://shouldiblockicmp.com/ - , "-A INPUT -p " + ipVersion.icmpProtocol() + " -j ACCEPT"); + // We reject with rules instead of using policies + rules.add("-P INPUT ACCEPT"); + rules.add("-P FORWARD ACCEPT"); + rules.add("-P OUTPUT ACCEPT"); + + // Allow packets belonging to established connections + rules.add( "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT"); + + // Allow any loopback traffic + rules.add("-A INPUT -i lo -j ACCEPT"); + + // Allow ICMP packets. See http://shouldiblockicmp.com/ + rules.add("-A INPUT -p " + ipVersion.icmpProtocol() + " -j ACCEPT"); // Allow trusted ports if any - String commaSeparatedPorts = trustedPorts.stream().map(i -> Integer.toString(i)).collect(Collectors.joining(",")); - String ports = commaSeparatedPorts.isEmpty() ? "" : "-A INPUT -p tcp -m multiport --dports " + commaSeparatedPorts + " -j ACCEPT\n"; + String commaSeparatedPorts = trustedPorts.stream().map(i -> Integer.toString(i)).sorted().collect(Collectors.joining(",")); + if (!commaSeparatedPorts.isEmpty()) + rules.add("-A INPUT -p tcp -m multiport --dports " + commaSeparatedPorts + " -j ACCEPT"); // Allow traffic from trusted nodes - String nodes = trustedNodes.stream() + trustedNodes.stream() .filter(ipVersion::match) .map(ipAddress -> "-A INPUT -s " + InetAddresses.toAddrString(ipAddress) + ipVersion.singleHostCidr() + " -j ACCEPT") - .collect(Collectors.joining("\n")); + .sorted() + .forEach(rules::add); // We reject instead of dropping to give us an easier time to figure out potential network issues - String rejectEverythingElse = "-A INPUT -j REJECT --reject-with " + ipVersion.icmpPortUnreachable(); + rules.add("-A INPUT -j REJECT --reject-with " + ipVersion.icmpPortUnreachable()); - return basics + "\n" + ports + nodes + "\n" + rejectEverythingElse; + return Collections.unmodifiableList(rules); } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java index 4e5906d3c34..611aff246d9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/FilterTableLineEditor.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEdit; import com.yahoo.vespa.hosted.node.admin.task.util.file.LineEditor; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; -import java.util.Arrays; import java.util.LinkedList; import java.util.List; @@ -16,17 +15,20 @@ class FilterTableLineEditor implements LineEditor { private final LinkedList<String> wantedRules; - FilterTableLineEditor(List<String> wantedRules) { + private FilterTableLineEditor(List<String> wantedRules) { this.wantedRules = new LinkedList<>(wantedRules); } static FilterTableLineEditor from(Acl acl, IPVersion ipVersion) { - List<String> rules = Arrays.asList(acl.toRules(ipVersion).split("\n")); + List<String> rules = acl.toRules(ipVersion); return new FilterTableLineEditor(rules); } @Override public LineEdit edit(String line) { + // We have already added all the lines we wanted, remove the remainer + if (wantedRules.isEmpty()) return LineEdit.remove(); + String wantedRule = wantedRules.pop(); return wantedRule.equals(line) ? LineEdit.none() : LineEdit.replaceWith(wantedRule); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditor.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditor.java index 7b33939398a..fe68ddef15e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditor.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/IPTablesEditor.java @@ -59,6 +59,7 @@ class IPTablesEditor { ProcessResult currentRulesResult = dockerOperations.executeCommandInNetworkNamespace(containerName, ipVersion.iptablesCmd(), "-S", "-t", table); return Arrays.stream(currentRulesResult.getOutput().split("\n")) + .map(String::trim) .collect(Collectors.toList()); }; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 2fccbc7038e..4b37678e376 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -232,7 +232,6 @@ public class NodeAgentImpl implements NodeAgent { if (! resumeScriptRun) { storageMaintainer.writeMetricsConfig(containerName, node); storageMaintainer.writeFilebeatConfig(containerName, node); - aclMaintainer.run(); stopFilebeatSchedulerIfNeeded(); currentFilebeatRestarter = Optional.of(filebeatRestarter.scheduleWithFixedDelay( () -> serviceRestarter.accept("filebeat"), 1, 1, TimeUnit.DAYS)); @@ -490,6 +489,7 @@ public class NodeAgentImpl implements NodeAgent { containerState = STARTING; startContainer(node); containerState = UNKNOWN; + aclMaintainer.run(); } runLocalResumeScriptIfNeeded(node); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java index 587e6212ad4..f9d0736fe21 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/provider/NodeAdminProvider.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.node.admin.provider; import com.google.inject.Inject; import com.yahoo.concurrent.classlock.ClassLocking; import com.yahoo.container.di.componentgraph.Provider; -import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.hosted.dockerapi.Docker; import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; @@ -22,8 +21,7 @@ public class NodeAdminProvider implements Provider<NodeAdminStateUpdater> { MetricReceiverWrapper metricReceiver, ClassLocking classLocking) { ConfigServerClients clients = new RealConfigServerClients( - new ConfigServerInfo(configServerConfig), - Defaults.getDefaults().vespaHostname()); + new ConfigServerInfo(configServerConfig)); dockerAdmin = new DockerAdminComponent(configServerConfig, docker, diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdaterTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdaterTest.java index 490f45b094c..6c6f8cf40fe 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdaterTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/SslConnectionSocketFactoryUpdaterTest.java @@ -1,58 +1,31 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver; -import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; -import com.yahoo.vespa.hosted.node.admin.configserver.certificate.ConfigServerKeyStoreRefresher; -import com.yahoo.vespa.hosted.node.admin.configserver.certificate.ConfigServerKeyStoreRefresherFactory; -import com.yahoo.vespa.hosted.node.admin.util.KeyStoreOptions; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.junit.Before; +import com.yahoo.vespa.athenz.identity.SiaIdentityProvider; +import com.yahoo.vespa.athenz.tls.SslContextBuilder; import org.junit.Test; -import java.util.Optional; - -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; +import static org.junit.Assert.assertNotNull; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; /** - * @author hakon + * @author bjorncs */ public class SslConnectionSocketFactoryUpdaterTest { - private final ConfigServerInfo configServerInfo = mock(ConfigServerInfo.class); - private final String hostname = "host.oath.com"; - private final ConfigServerKeyStoreRefresherFactory refresherFactory = - mock(ConfigServerKeyStoreRefresherFactory.class); - private final ConfigServerKeyStoreRefresher refresher = - mock(ConfigServerKeyStoreRefresher.class); - private final SslConnectionSocketFactoryCreator socketFactoryCreator = - mock(SslConnectionSocketFactoryCreator.class); - private final SSLConnectionSocketFactory socketFactory = mock(SSLConnectionSocketFactory.class); - @Before - public void setUp() { - KeyStoreOptions keyStoreOptions = mock(KeyStoreOptions.class); - when(configServerInfo.getKeyStoreOptions()).thenReturn(Optional.of(keyStoreOptions)); - when(refresherFactory.create(any(), any(), any(), any())).thenReturn(refresher); - when(socketFactoryCreator.createSocketFactory(any(), any())) - .thenReturn(socketFactory); + @Test + public void creates_default_ssl_connection_factory_when_no_sia_provided() { + SslConnectionSocketFactoryUpdater updater = + new SslConnectionSocketFactoryUpdater(null, (hostname, session) -> true); + assertNotNull(updater.getCurrentSocketFactory()); } @Test - public void testSettingOfSocketFactory() { - SslConnectionSocketFactoryUpdater updater = new SslConnectionSocketFactoryUpdater( - configServerInfo, - hostname, - refresherFactory, - socketFactoryCreator); - - assertTrue(socketFactory == updater.getCurrentSocketFactory()); - - ConfigServerApi api = mock(ConfigServerApi.class); - updater.registerConfigServerApi(api); - verify(api, times(1)).setSSLConnectionSocketFactory(socketFactory); + public void creates_ssl_connection_factory_when_sia_provided() { + SiaIdentityProvider sia = mock(SiaIdentityProvider.class); + when(sia.getIdentitySslContext()).thenReturn(new SslContextBuilder().build()); + SslConnectionSocketFactoryUpdater updater = new SslConnectionSocketFactoryUpdater(sia, (hostname, session) -> true); + assertNotNull(updater.getCurrentSocketFactory()); } }
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java index 181928fa438..77bc49ca596 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/AclTest.java @@ -1,7 +1,6 @@ -package com.yahoo.vespa.hosted.node.admin.maintenance.acl; +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; import com.google.common.net.InetAddresses; -import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Acl; import com.yahoo.vespa.hosted.node.admin.task.util.network.IPVersion; import org.junit.Assert; import org.junit.Test; @@ -16,7 +15,7 @@ public class AclTest { private final Acl aclCommon = new Acl( createPortList(1234, 453), - createTrustedNodes("192.1.2.2", "fb00::1", "fe80::2")); + createTrustedNodes("192.1.2.2", "fb00::1", "fe80::2", "fe80::3")); private final Acl aclNoPorts = new Acl( Collections.emptyList(), @@ -24,7 +23,7 @@ public class AclTest { @Test public void no_trusted_ports() { - String listRulesIpv4 = aclNoPorts.toRules(IPVersion.IPv4); + String listRulesIpv4 = String.join("\n", aclNoPorts.toRules(IPVersion.IPv4)); Assert.assertEquals( "-P INPUT ACCEPT\n" + "-P FORWARD ACCEPT\n" + @@ -39,7 +38,7 @@ public class AclTest { @Test public void ipv4_list_rules() { - String listRulesIpv4 = aclCommon.toRules(IPVersion.IPv4); + String listRulesIpv4 = String.join("\n", aclCommon.toRules(IPVersion.IPv4)); Assert.assertEquals( "-P INPUT ACCEPT\n" + "-P FORWARD ACCEPT\n" + @@ -55,7 +54,7 @@ public class AclTest { @Test public void ipv6_list_rules() { - String listRulesIpv6 = aclCommon.toRules(IPVersion.IPv6); + String listRulesIpv6 = String.join("\n", aclCommon.toRules(IPVersion.IPv6)); Assert.assertEquals( "-P INPUT ACCEPT\n" + "-P FORWARD ACCEPT\n" + @@ -66,9 +65,21 @@ public class AclTest { "-A INPUT -p tcp -m multiport --dports 1234,453 -j ACCEPT\n" + "-A INPUT -s fb00::1/128 -j ACCEPT\n" + "-A INPUT -s fe80::2/128 -j ACCEPT\n" + + "-A INPUT -s fe80::3/128 -j ACCEPT\n" + "-A INPUT -j REJECT --reject-with icmp6-port-unreachable", listRulesIpv6); } + @Test + public void ipv6_rules_stable() { + Acl aclCommonDifferentOrder = new Acl( + createPortList(453, 1234), + createTrustedNodes("fe80::2", "192.1.2.2", "fb00::1", "fe80::3")); + + for (IPVersion ipVersion: IPVersion.values()) { + Assert.assertEquals(aclCommon.toRules(ipVersion), aclCommonDifferentOrder.toRules(ipVersion)); + } + } + private List<Integer> createPortList(Integer... ports) { return Arrays.asList(ports); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java index 0efc84727fc..28e21494c01 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/acl/AclMaintainerTest.java @@ -131,7 +131,7 @@ public class AclMaintainerTest { "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + "-A INPUT -i lo -j ACCEPT\n" + "-A INPUT -p ipv6-icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 4321,2345,22 -j ACCEPT\n" + + "-A INPUT -p tcp -m multiport --dports 22,2345,4321 -j ACCEPT\n" + "-A INPUT -s 2001::1/128 -j ACCEPT\n" + "-A INPUT -s fd01:1234::4321/128 -j ACCEPT\n" + "-A INPUT -j REJECT --reject-with icmp6-port-unreachable"; @@ -165,7 +165,7 @@ public class AclMaintainerTest { "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + "-A INPUT -i lo -j ACCEPT\n" + "-A INPUT -p icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22,4443,2222 -j ACCEPT\n" + + "-A INPUT -p tcp -m multiport --dports 22,2222,4443 -j ACCEPT\n" + "-A INPUT -s 192.64.13.2/32 -j ACCEPT\n" + "-A INPUT -j REJECT --reject-with icmp-port-unreachable"; @@ -175,7 +175,7 @@ public class AclMaintainerTest { "-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\n" + "-A INPUT -i lo -j ACCEPT\n" + "-A INPUT -p ipv6-icmp -j ACCEPT\n" + - "-A INPUT -p tcp -m multiport --dports 22,4443,2222 -j ACCEPT\n" + + "-A INPUT -p tcp -m multiport --dports 22,2222,4443 -j ACCEPT\n" + "-A INPUT -s 2001::1/128 -j ACCEPT\n" + "-A INPUT -j REJECT --reject-with icmp6-port-unreachable"; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java index a936beac797..b33d776ac08 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTest.java @@ -18,9 +18,9 @@ import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; -import static junit.framework.TestCase.fail; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; /** * tests basic operation of the node repository @@ -178,4 +178,5 @@ public class NodeRepositoryTest { .map(Node::hostname) .collect(Collectors.toSet()); } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java index dce9f694647..1e69b26d0e0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DockerHostCapacityTest.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; @@ -16,9 +15,9 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author smorgrav @@ -28,8 +27,8 @@ public class DockerHostCapacityTest { private DockerHostCapacity capacity; private List<Node> nodes; private Node host1, host2, host3; - Node nodeA, nodeB, nodeC, nodeD, nodeE; - Flavor flavorDocker, flavorDocker2; + private Node nodeA, nodeB, nodeC, nodeD, nodeE; + private Flavor flavorDocker, flavorDocker2; @Before public void setup() { @@ -132,10 +131,4 @@ public class DockerHostCapacityTest { return additionalIps; } - private ApplicationId app(String tenant) { - return new ApplicationId.Builder() - .tenant(tenant) - .applicationName("test") - .instanceName("default").build(); - } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java index de2aaf68788..e885274c154 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/v2/filter/NodeIdentifierTest.java @@ -34,14 +34,15 @@ import static com.yahoo.vespa.athenz.tls.SignatureAlgorithm.SHA256_WITH_RSA; import static java.util.Collections.emptySet; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; -import static junit.framework.TestCase.assertFalse; -import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author bjorncs */ public class NodeIdentifierTest { + private static final String HOSTNAME = "myhostname"; private static final String OPENSTACK_ID = "OPENSTACK-ID"; private static final String AWS_INSTANCE_ID = "i-abcdef123456"; @@ -188,4 +189,5 @@ public class NodeIdentifierTest { b.addFlavor("docker", 1., 2., 50, Flavor.Type.DOCKER_CONTAINER).cost(1); return b.build(); } + }
\ No newline at end of file diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/VespaModelUtilTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/VespaModelUtilTest.java index 33efc2a8486..537101e6c51 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/VespaModelUtilTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/model/VespaModelUtilTest.java @@ -21,15 +21,16 @@ import java.util.Optional; import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceClusterSet; import static com.yahoo.vespa.orchestrator.TestUtil.makeServiceInstanceSet; -import static junit.framework.TestCase.assertFalse; -import static junit.framework.TestCase.assertTrue; -import static org.fest.assertions.Assertions.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** * @author hakonhall */ public class VespaModelUtilTest { + // Cluster Controller Service Cluster private static final ClusterId CONTENT_CLUSTER_ID = new ClusterId("content-cluster-0"); @@ -174,25 +175,25 @@ public class VespaModelUtilTest { List<HostName> controllers = VespaModelUtil.getClusterControllerInstancesInOrder(application, CONTENT_CLUSTER_ID); List<HostName> expectedControllers = Arrays.asList(controller0.hostName(), controller1.hostName()); - assertThat(controllers).isEqualTo(expectedControllers); + assertEquals(expectedControllers, controllers); } @Test public void testGetControllerHostName() { HostName host = VespaModelUtil.getControllerHostName(application, CONTENT_CLUSTER_ID); - assertThat(host).isEqualTo(controller0Host); + assertEquals(controller0Host, host); } @Test public void testGetContentClusterName() { ClusterId contentClusterName = VespaModelUtil.getContentClusterName(application, distributor0.hostName()); - assertThat(CONTENT_CLUSTER_ID).isEqualTo(contentClusterName); + assertEquals(CONTENT_CLUSTER_ID, contentClusterName); } @Test public void testGetContentClusterNameForSecondaryContentCluster() { ClusterId contentClusterName = VespaModelUtil.getContentClusterName(application, secondaryDistributor0.hostName()); - assertThat(SECONDARY_CONTENT_CLUSTER_ID).isEqualTo(contentClusterName); + assertEquals(contentClusterName, SECONDARY_CONTENT_CLUSTER_ID); } @Test @@ -200,7 +201,7 @@ public class VespaModelUtilTest { Optional<ServiceInstance> service = VespaModelUtil.getStorageNodeAtHost(application, storage0Host); assertTrue(service.isPresent()); - assertThat(service.get()).isEqualTo(storage0); + assertEquals(storage0, service.get()); } @Test @@ -213,13 +214,13 @@ public class VespaModelUtilTest { @Test public void testGetClusterControllerIndex() { ConfigId configId = new ConfigId("admin/cluster-controllers/2"); - assertThat(VespaModelUtil.getClusterControllerIndex(configId)).isEqualTo(2); + assertEquals(2, VespaModelUtil.getClusterControllerIndex(configId)); } @Test public void testGetClusterControllerIndexWithStandaloneClusterController() { ConfigId configId = new ConfigId("fantasy_sports/standalone/fantasy_sports-controllers/1"); - assertThat(VespaModelUtil.getClusterControllerIndex(configId)).isEqualTo(1); + assertEquals(1, VespaModelUtil.getClusterControllerIndex(configId)); } @Test(expected = IllegalArgumentException.class) @@ -232,6 +233,7 @@ public class VespaModelUtilTest { @Test public void testGetStorageNodeIndex() { ConfigId configId = TestUtil.storageNodeConfigId(CONTENT_CLUSTER_ID.toString(), 3); - assertThat(VespaModelUtil.getStorageNodeIndex(configId)).isEqualTo(3); + assertEquals(3, VespaModelUtil.getStorageNodeIndex(configId)); } + } diff --git a/predicate-search/src/test/java/com/yahoo/search/predicate/index/IntervalPostingListTest.java b/predicate-search/src/test/java/com/yahoo/search/predicate/index/IntervalPostingListTest.java index f9f65034cc6..db1ae5b1fa2 100644 --- a/predicate-search/src/test/java/com/yahoo/search/predicate/index/IntervalPostingListTest.java +++ b/predicate-search/src/test/java/com/yahoo/search/predicate/index/IntervalPostingListTest.java @@ -6,11 +6,12 @@ import org.junit.Test; import java.util.Arrays; -import static junit.framework.TestCase.assertFalse; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class IntervalPostingListTest { + @Test public void requireThatPostingListCanIterate() { PredicateIntervalStore.Builder builder = new PredicateIntervalStore.Builder(); diff --git a/processing/src/test/java/com/yahoo/processing/execution/test/ExecutionContextTestCase.java b/processing/src/test/java/com/yahoo/processing/execution/test/ExecutionContextTestCase.java index 5ee1955a066..9c4d4de47dc 100644 --- a/processing/src/test/java/com/yahoo/processing/execution/test/ExecutionContextTestCase.java +++ b/processing/src/test/java/com/yahoo/processing/execution/test/ExecutionContextTestCase.java @@ -5,19 +5,26 @@ import com.yahoo.component.chain.Chain; import com.yahoo.processing.Processor; import com.yahoo.processing.execution.Execution; import com.yahoo.processing.test.ProcessorLibrary; +import org.junit.Test; import java.util.HashSet; import java.util.Iterator; import java.util.Set; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + /** - * @author bratseth + * @author bratseth */ -public class ExecutionContextTestCase extends junit.framework.TestCase { +public class ExecutionContextTestCase { - private Chain<Processor> chain=new Chain<Processor>(new ProcessorLibrary.DataSource()); + private final Chain<Processor> chain = new Chain<Processor>(new ProcessorLibrary.DataSource()); /** Tests combined use of trace messages, context values and access log entries */ + @Test public void testtrace() { Execution execution1=Execution.createRoot(chain,2,Execution.Environment.createEmpty()); execution1.trace().setProperty("a","a1"); diff --git a/processing/src/test/java/com/yahoo/processing/request/test/ErrorMessageTestCase.java b/processing/src/test/java/com/yahoo/processing/request/test/ErrorMessageTestCase.java index 334c7afb076..70f4a7720ee 100644 --- a/processing/src/test/java/com/yahoo/processing/request/test/ErrorMessageTestCase.java +++ b/processing/src/test/java/com/yahoo/processing/request/test/ErrorMessageTestCase.java @@ -1,14 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.processing.request.test; -import com.yahoo.processing.Request; import com.yahoo.processing.request.ErrorMessage; import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + /** * @author bratseth */ -public class ErrorMessageTestCase extends junit.framework.TestCase { +public class ErrorMessageTestCase { @Test public void testToString() { @@ -44,11 +46,16 @@ public class ErrorMessageTestCase extends junit.framework.TestCase { new ErrorMessage("message")); assertEquals(new ErrorMessage("message",new Exception()), new ErrorMessage("message")); - assertFalse(new ErrorMessage("message").equals(new ErrorMessage("message","detail"))); - assertFalse(new ErrorMessage(37,"message").equals(new ErrorMessage("message"))); - assertFalse(new ErrorMessage(37,"message").equals(new ErrorMessage(38,"message"))); - assertFalse(new ErrorMessage("message","detail1").equals(new ErrorMessage("message","detail2"))); - assertFalse(new ErrorMessage("message1").equals(new ErrorMessage("message2"))); + assertNotEquals(new ErrorMessage("message"), + new ErrorMessage("message","detail")); + assertNotEquals(new ErrorMessage(37,"message"), + new ErrorMessage("message")); + assertNotEquals(new ErrorMessage(37,"message"), + new ErrorMessage(38,"message")); + assertNotEquals(new ErrorMessage("message","detail1"), + new ErrorMessage("message","detail2")); + assertNotEquals(new ErrorMessage("message1"), + new ErrorMessage("message2")); } } diff --git a/processing/src/test/java/com/yahoo/processing/request/test/RequestTestCase.java b/processing/src/test/java/com/yahoo/processing/request/test/RequestTestCase.java index 100e4f3781f..032fdd71f88 100644 --- a/processing/src/test/java/com/yahoo/processing/request/test/RequestTestCase.java +++ b/processing/src/test/java/com/yahoo/processing/request/test/RequestTestCase.java @@ -8,12 +8,17 @@ import com.yahoo.processing.request.Properties; import com.yahoo.processing.request.properties.PropertyMap; import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + /** * Tests using requests * * @author bratseth */ -public class RequestTestCase extends junit.framework.TestCase { +public class RequestTestCase { + + private static final double delta = 0.0000000001; @Test public void testProperties() { @@ -33,53 +38,53 @@ public class RequestTestCase extends junit.framework.TestCase { assertEquals("default", r.properties().get(new CompoundName("c"), "default")); assertNull(r.properties().get(new CompoundName("c"))); - assertEquals("b1",r.properties().getString("b")); - assertEquals("b1",r.properties().getString("b","default")); - assertEquals("default",r.properties().getString("c","default")); - assertEquals(null,r.properties().getString("c")); - assertEquals("b1",r.properties().getString(new CompoundName("b"))); - assertEquals("b1",r.properties().getString(new CompoundName("b"),"default")); - assertEquals("default",r.properties().getString(new CompoundName("c"),"default")); - assertEquals(null,r.properties().getString(new CompoundName("c"))); + assertEquals("b1", r.properties().getString("b")); + assertEquals("b1", r.properties().getString("b","default")); + assertEquals("default", r.properties().getString("c","default")); + assertEquals(null, r.properties().getString("c")); + assertEquals("b1", r.properties().getString(new CompoundName("b"))); + assertEquals("b1", r.properties().getString(new CompoundName("b"),"default")); + assertEquals("default", r.properties().getString(new CompoundName("c"),"default")); + assertEquals(null, r.properties().getString(new CompoundName("c"))); r.properties().set("i",7); - assertEquals(7,(int)r.properties().getInteger("i")); - assertEquals(7,(int)r.properties().getInteger("i",3)); - assertEquals(3,(int)r.properties().getInteger("n",3)); + assertEquals(7, (int)r.properties().getInteger("i")); + assertEquals(7, (int)r.properties().getInteger("i",3)); + assertEquals(3, (int)r.properties().getInteger("n",3)); assertNull(r.properties().getInteger("n")); - assertEquals(7,(int)r.properties().getInteger(new CompoundName("i"))); - assertEquals(7,(int)r.properties().getInteger(new CompoundName("i"),3)); - assertEquals(3,(int)r.properties().getInteger(new CompoundName("n"),3)); + assertEquals(7, (int)r.properties().getInteger(new CompoundName("i"))); + assertEquals(7, (int)r.properties().getInteger(new CompoundName("i"),3)); + assertEquals(3, (int)r.properties().getInteger(new CompoundName("n"),3)); assertNull(r.properties().getInteger("n")); - r.properties().set(new CompoundName("l"),7); + r.properties().set(new CompoundName("l"), 7); assertEquals(7, (long) r.properties().getLong("l")); - assertEquals(7,(long)r.properties().getLong("l",3l)); - assertEquals(3,(long)r.properties().getLong("m",3l)); + assertEquals(7, (long)r.properties().getLong("l",3l)); + assertEquals(3, (long)r.properties().getLong("m",3l)); assertNull(r.properties().getInteger("m")); - assertEquals(7,(long)r.properties().getLong(new CompoundName("l"))); - assertEquals(7,(long)r.properties().getLong(new CompoundName("l"),3l)); - assertEquals(3,(long)r.properties().getLong(new CompoundName("m"),3l)); + assertEquals(7, (long)r.properties().getLong(new CompoundName("l"))); + assertEquals(7, (long)r.properties().getLong(new CompoundName("l"),3l)); + assertEquals(3, (long)r.properties().getLong(new CompoundName("m"),3l)); assertNull(r.properties().getInteger("m")); - r.properties().set("d",7.3); - assertEquals(7.3,r.properties().getDouble("d")); - assertEquals(7.3,r.properties().getDouble("d",3.4d)); - assertEquals(3.4,r.properties().getDouble("f",3.4d)); + r.properties().set("d", 7.3); + assertEquals(7.3, r.properties().getDouble("d"), delta); + assertEquals(7.3, r.properties().getDouble("d",3.4d), delta); + assertEquals(3.4, r.properties().getDouble("f",3.4d), delta); assertNull(r.properties().getDouble("f")); - assertEquals(7.3,r.properties().getDouble(new CompoundName("d"))); - assertEquals(7.3,r.properties().getDouble(new CompoundName("d"),3.4d)); - assertEquals(3.4,r.properties().getDouble(new CompoundName("f"),3.4d)); + assertEquals(7.3, r.properties().getDouble(new CompoundName("d")), delta); + assertEquals(7.3, r.properties().getDouble(new CompoundName("d"),3.4d), delta); + assertEquals(3.4, r.properties().getDouble(new CompoundName("f"),3.4d), delta); assertNull(r.properties().getDouble("f")); r.properties().set("o",true); - assertEquals(true,r.properties().getBoolean("o")); - assertEquals(true,r.properties().getBoolean("o",true)); - assertEquals(true,r.properties().getBoolean("g",true)); + assertEquals(true, r.properties().getBoolean("o")); + assertEquals(true, r.properties().getBoolean("o",true)); + assertEquals(true, r.properties().getBoolean("g",true)); assertEquals(false, r.properties().getBoolean("g")); - assertEquals(true,r.properties().getBoolean(new CompoundName("o"))); - assertEquals(true,r.properties().getBoolean(new CompoundName("o"),true)); - assertEquals(true,r.properties().getBoolean(new CompoundName("g"),true)); + assertEquals(true, r.properties().getBoolean(new CompoundName("o"))); + assertEquals(true, r.properties().getBoolean(new CompoundName("o"),true)); + assertEquals(true, r.properties().getBoolean(new CompoundName("g"),true)); assertEquals(false, r.properties().getBoolean("g")); r.properties().set(new CompoundName("x.y"), "x1.y1"); @@ -99,9 +104,8 @@ public class RequestTestCase extends junit.framework.TestCase { r.errors().add(new ErrorMessage("foo")); r.errors().add(new ErrorMessage("bar")); assertEquals(2,r.errors().size()); - assertEquals("foo",r.errors().get(0).getMessage()); - assertEquals("bar",r.errors().get(1).getMessage()); - + assertEquals("foo", r.errors().get(0).getMessage()); + assertEquals("bar", r.errors().get(1).getMessage()); } @Test diff --git a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp index 34de8a2e6a3..c333b8943e2 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistence_handler_map/persistence_handler_map_test.cpp @@ -127,16 +127,6 @@ TEST_F("require that handler snapshot can be retrieved for given bucket space", TEST_DO(assertSnapshot({}, f.map.getHandlerSnapshot(space_null))); } -TEST_F("require that handler snapshot can be retrieved for given document type (in bucket space)", Fixture) -{ - // Note: Document id doesn't contain document type -> all handlers returned - TEST_DO(assertSnapshot({handler_a, handler_b}, - f.map.getHandlerSnapshot(space_1, DocumentId("userdoc:namespace:1234:namespace")))); - TEST_DO(assertSnapshot({handler_a}, - f.map.getHandlerSnapshot(space_1, DocumentId("id:namespace:a::doc1")))); - EXPECT_TRUE(f.map.getHandlerSnapshot(space_1, DocumentId("id:namespace:c::doc2")).get() == nullptr); -} - TEST_MAIN() { TEST_RUN_ALL(); diff --git a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp index a016a56edf7..9262f9a7b6f 100644 --- a/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp +++ b/searchcore/src/tests/proton/persistenceengine/persistenceengine_test.cpp @@ -565,12 +565,15 @@ TEST_F("require that removes are routed to handlers", SimpleFixture) assertHandler(bucket0, tstamp0, docId0, f.hset.handler1); assertHandler(bucket0, tstamp0, docId0, f.hset.handler2); EXPECT_FALSE(rr.wasFound()); + EXPECT_TRUE(rr.hasError()); + EXPECT_EQUAL(Result(Result::PERMANENT_ERROR, "No handler for document type 'type3'"), rr); f.hset.handler1.setExistingTimestamp(tstamp2); rr = f.engine.remove(bucket1, tstamp1, docId1, context); assertHandler(bucket1, tstamp1, docId1, f.hset.handler1); assertHandler(bucket0, tstamp0, docId0, f.hset.handler2); EXPECT_TRUE(rr.wasFound()); + EXPECT_FALSE(rr.hasError()); f.hset.handler1.setExistingTimestamp(tstamp0); f.hset.handler2.setExistingTimestamp(tstamp3); @@ -578,12 +581,14 @@ TEST_F("require that removes are routed to handlers", SimpleFixture) assertHandler(bucket1, tstamp1, docId1, f.hset.handler1); assertHandler(bucket1, tstamp1, docId2, f.hset.handler2); EXPECT_TRUE(rr.wasFound()); + EXPECT_FALSE(rr.hasError()); f.hset.handler2.setExistingTimestamp(tstamp0); rr = f.engine.remove(bucket1, tstamp1, docId2, context); assertHandler(bucket1, tstamp1, docId1, f.hset.handler1); assertHandler(bucket1, tstamp1, docId2, f.hset.handler2); EXPECT_FALSE(rr.wasFound()); + EXPECT_FALSE(rr.hasError()); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp index cf7d027f244..feea15ddadd 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp @@ -94,18 +94,4 @@ public: } -HandlerSnapshot::UP -PersistenceHandlerMap::getHandlerSnapshot(document::BucketSpace bucketSpace, - const document::DocumentId &id) const -{ - if (!id.hasDocType()) { - return getHandlerSnapshot(bucketSpace); - } - IPersistenceHandler::SP handler = getHandler(bucketSpace, DocTypeName(id.getDocType())); - if (!handler.get()) { - return HandlerSnapshot::UP(); - } - return std::make_unique<HandlerSnapshot>(SequenceOfOne::make(handler.get()), 1); -} - } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h index 8a852066284..003c378d6a7 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h @@ -62,8 +62,6 @@ public: const DocTypeName &docType) const; HandlerSnapshot::UP getHandlerSnapshot() const; HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace) const; - HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace, - const document::DocumentId &id) const; }; } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 0950a596f5b..056497d137e 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -173,13 +173,6 @@ PersistenceEngine::getHandlerSnapshot(document::BucketSpace bucketSpace) const return _handlers.getHandlerSnapshot(bucketSpace); } -PersistenceEngine::HandlerSnapshot::UP -PersistenceEngine::getHandlerSnapshot(document::BucketSpace bucketSpace, const DocumentId &id) const -{ - std::lock_guard<std::mutex> guard(_lock); - return _handlers.getHandlerSnapshot(bucketSpace, id); -} - PersistenceEngine::PersistenceEngine(IPersistenceEngineOwner &owner, const IResourceWriteFilter &writeFilter, ssize_t defaultSerializedSize, bool ignoreMaxBytes) : AbstractPersistenceProvider(), @@ -356,17 +349,17 @@ PersistenceEngine::RemoveResult PersistenceEngine::remove(const Bucket& b, Timestamp t, const DocumentId& did, Context&) { std::shared_lock<std::shared_timed_mutex> rguard(_rwMutex); + assert(did.hasDocType()); + DocTypeName docType(did.getDocType()); LOG(spam, "remove(%s, %" PRIu64 ", \"%s\")", b.toString().c_str(), static_cast<uint64_t>(t.getValue()), did.toString().c_str()); - HandlerSnapshot::UP snap = getHandlerSnapshot(b.getBucketSpace(), did); - if (!snap) { - return RemoveResult(false); - } - TransportLatch latch(snap->size()); - for (; snap->handlers().valid(); snap->handlers().next()) { - IPersistenceHandler *handler = snap->handlers().get(); - handler->handleRemove(feedtoken::make(latch), b, t, did); + IPersistenceHandler::SP handler = getHandler(b.getBucketSpace(), docType); + if (!handler) { + return RemoveResult(Result::PERMANENT_ERROR, + make_string("No handler for document type '%s'", docType.toString().c_str())); } + TransportLatch latch(1); + handler->handleRemove(feedtoken::make(latch), b, t, did); latch.await(); return latch.getRemoveResult(); } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index a9c15e6f2b9..f0e7b7057de 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -84,8 +84,6 @@ private: const DocTypeName &docType) const; HandlerSnapshot::UP getHandlerSnapshot() const; HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace) const; - HandlerSnapshot::UP getHandlerSnapshot(document::BucketSpace bucketSpace, - const document::DocumentId &docId) const; void saveClusterState(BucketSpace bucketSpace, const ClusterState &calc); ClusterState::SP savedClusterState(BucketSpace bucketSpace) const; diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/AggregationTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/AggregationTestCase.java index 1c0212babf8..cbd6c02cd26 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/AggregationTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/AggregationTestCase.java @@ -9,13 +9,19 @@ import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.Identifiable; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; -import junit.framework.TestCase; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; /** * @author baldersheim */ -public class AggregationTestCase extends TestCase { +public class AggregationTestCase { + + private static final double delta = 0.0000000001; + @Test public void testSumAggregationResult() { SumAggregationResult a = new SumAggregationResult(); a.setExpression(new AttributeNode("attributeA")); @@ -27,6 +33,7 @@ public class AggregationTestCase extends TestCase { assertEquals(b.getSum().getInteger(), 14); } + @Test public void testXorAggregationResult() { XorAggregationResult a = new XorAggregationResult(6); a.setExpression(new AttributeNode("attributeA")); @@ -39,6 +46,7 @@ public class AggregationTestCase extends TestCase { assertEquals(b.getXor(), 0); } + @Test public void testCountAggregationResult() { CountAggregationResult a = new CountAggregationResult(6); a.setExpression(new AttributeNode("attributeA")); @@ -51,6 +59,7 @@ public class AggregationTestCase extends TestCase { assertEquals(b.getCount(), 14); } + @Test public void testMinAggregationResult() { MinAggregationResult a = new MinAggregationResult(new IntegerResultNode(6)); a.setExpression(new AttributeNode("attributeA")); @@ -64,6 +73,7 @@ public class AggregationTestCase extends TestCase { assertEquals(b.getMin().getInteger(), 6); } + @Test public void testMaxAggregationResult() { MaxAggregationResult a = new MaxAggregationResult(new IntegerResultNode(6)); a.setExpression(new AttributeNode("attributeA")); @@ -77,6 +87,7 @@ public class AggregationTestCase extends TestCase { assertEquals(b.getMax().getInteger(), 7); } + @Test public void testAverageAggregationResult() { AverageAggregationResult a = new AverageAggregationResult(new FloatResultNode(72), 6); a.setExpression(new AttributeNode("attributeA")); @@ -99,6 +110,7 @@ public class AggregationTestCase extends TestCase { return new GlobalId((new DocumentId("doc:test:" + docId)).getGlobalId()); } + @Test public void testFs4HitsAggregationResult() { double rank1 = 1; double rank2 = 2; @@ -118,10 +130,10 @@ public class AggregationTestCase extends TestCase { assertEquals(a, b); a.postMerge(); assertEquals(2, a.getHits().size()); - assertEquals(2.0, a.getHits().get(0).getRank()); + assertEquals(2.0, a.getHits().get(0).getRank(), delta); a.setMaxHits(1).postMerge(); assertEquals(1, a.getHits().size()); - assertEquals(2.0, a.getHits().get(0).getRank()); + assertEquals(2.0, a.getHits().get(0).getRank(), delta); HitsAggregationResult hits = new HitsAggregationResult(3) .addHit(new FS4Hit(1, createGlobalId(3), 1)) @@ -160,6 +172,7 @@ public class AggregationTestCase extends TestCase { assertFS4Hits(request, 4, 5, 0); } + @Test public void testVdsHitsAggregationResult() { double rank1 = 1; double rank2 = 2; @@ -176,8 +189,6 @@ public class AggregationTestCase extends TestCase { assertEquals(0, a.getHits().size()); a.setExpression(new AttributeNode("attributeA")); a.addHit(new VdsHit("1", s2, rank1)); -// a.addHit(new VdsHit("5", s7, rank2)); -// assertEquals(2, a.getHits().size()); HitsAggregationResult b = (HitsAggregationResult)serializeDeserialize(a); assertEquals(a, b); @@ -218,7 +229,6 @@ public class AggregationTestCase extends TestCase { assertVdsHits(request, 4, 5, 0); } - private void assertFS4Hits(Grouping request, int firstLevel, int lastLevel, int expected) { CountFS4Hits obj = new CountFS4Hits(); request.setFirstLevel(firstLevel); @@ -255,6 +265,7 @@ public class AggregationTestCase extends TestCase { } } + @Test public void testGroup() { Group a = new Group(); a.setId(new IntegerResultNode(17)); @@ -262,6 +273,7 @@ public class AggregationTestCase extends TestCase { serializeDeserialize1(a); } + @Test public void testGrouping() { Grouping a = new Grouping(); GroupingLevel level = new GroupingLevel(); @@ -293,7 +305,6 @@ public class AggregationTestCase extends TestCase { a.getRoot().addChild(g); serializeDeserialize1(a); - Grouping foo = new Grouping(); foo.addLevel(level); int hashBefore = foo.hashCode(); @@ -320,7 +331,7 @@ public class AggregationTestCase extends TestCase { buf.flip(); Identifiable b = Identifiable.create(buf); assertEquals(a.getClass(), b.getClass()); - assertEquals(buf.getBuf().hasRemaining(), false); + assertFalse(buf.getBuf().hasRemaining()); Identifiable c = b.clone(); assertEquals(b.getClass(), c.getClass()); BufferSerializer bb = new BufferSerializer(new GrowableByteBuffer()); @@ -343,4 +354,5 @@ public class AggregationTestCase extends TestCase { assertEquals(a.getExpression().getClass(), b.getExpression().getClass()); return b; } + } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/ForceLoadTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/ForceLoadTestCase.java index f2abc2069d7..347c38898ee 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/ForceLoadTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/ForceLoadTestCase.java @@ -1,19 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.aggregation; -public class ForceLoadTestCase extends junit.framework.TestCase { +import org.junit.Test; - public ForceLoadTestCase(String name) { - super(name); - } +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class ForceLoadTestCase { + @Test public void testLoadClasses() { try { new com.yahoo.searchlib.aggregation.ForceLoad(); assertTrue(com.yahoo.searchlib.aggregation.ForceLoad.forceLoad()); } catch (com.yahoo.system.ForceLoadError e) { e.printStackTrace(); - assertTrue(false); + fail("Load failed"); } } + } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/MergeTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/MergeTestCase.java index f94806fb00a..83b7e81a539 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/aggregation/MergeTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/aggregation/MergeTestCase.java @@ -3,22 +3,32 @@ package com.yahoo.searchlib.aggregation; import com.yahoo.document.DocumentId; import com.yahoo.document.GlobalId; -import com.yahoo.searchlib.expression.*; -import junit.framework.TestCase; +import com.yahoo.searchlib.expression.AggregationRefNode; +import com.yahoo.searchlib.expression.AttributeNode; +import com.yahoo.searchlib.expression.ConstantNode; +import com.yahoo.searchlib.expression.FloatBucketResultNode; +import com.yahoo.searchlib.expression.FloatResultNode; +import com.yahoo.searchlib.expression.IntegerResultNode; +import com.yahoo.searchlib.expression.MultiplyFunctionNode; +import com.yahoo.searchlib.expression.StringResultNode; +import org.junit.Test; import java.util.Arrays; import java.util.List; +import static org.junit.Assert.assertEquals; + /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ -public class MergeTestCase extends TestCase { +public class MergeTestCase { private GlobalId createGlobalId(int docId) { return new GlobalId((new DocumentId("doc:test:" + docId)).getGlobalId()); } // Test merging of hits. + @Test public void testMergeHits() { Grouping request = new Grouping() .setFirstLevel(0) @@ -33,7 +43,7 @@ public class MergeTestCase extends TestCase { .addHit(new FS4Hit(10, createGlobalId(10), 10)) .addHit(new FS4Hit(5, createGlobalId(9), 9)) .addHit(new FS4Hit(6, createGlobalId(8), 8)) - .setExpression(new ConstantNode( new IntegerResultNode(0)))); + .setExpression(new ConstantNode(new IntegerResultNode(0)))); Group a = new Group() .addAggregationResult(new HitsAggregationResult() @@ -68,6 +78,7 @@ public class MergeTestCase extends TestCase { } // Test merging the sum of the values from a single attribute vector that was collected directly into the root node. + @Test public void testMergeSimpleSum() { Grouping lhs = new Grouping() .setRoot(new Group() @@ -90,6 +101,7 @@ public class MergeTestCase extends TestCase { } // Test merging of the value from a single attribute vector in level 1. + @Test public void testMergeSingleChild() { Grouping lhs = new Grouping() .setFirstLevel(0) @@ -119,6 +131,7 @@ public class MergeTestCase extends TestCase { } // Test merging of the value from a multiple attribute vectors in level 1. + @Test public void testMergeMultiChild() { Grouping lhs = new Grouping() .setFirstLevel(0) @@ -171,6 +184,7 @@ public class MergeTestCase extends TestCase { } // Verify that frozen levels are not touched during merge. + @Test public void testMergeLevels() { Grouping request = new Grouping() .addLevel(new GroupingLevel() @@ -340,6 +354,7 @@ public class MergeTestCase extends TestCase { // Verify that the number of groups for a level is pruned down to maxGroups, that the remaining groups are the // highest ranked ones, and that they are sorted by group id. + @Test public void testMergeGroups() { Grouping request = new Grouping() .addLevel(new GroupingLevel() @@ -393,6 +408,7 @@ public class MergeTestCase extends TestCase { assertMerge(request, rhs, lhs, expectAll); } + @Test public void testMergeBuckets() { Grouping lhs = new Grouping() .setRoot(new Group().setTag(0) @@ -423,6 +439,7 @@ public class MergeTestCase extends TestCase { } // Merge two trees that are ordered by an expression, and verify that the resulting order after merge is correct. + @Test public void testMergeExpressions() { Grouping a = new Grouping() .setFirstLevel(0) @@ -465,6 +482,7 @@ public class MergeTestCase extends TestCase { } // Merge two relatively complex tree structures and verify that the end result is as expected. + @Test public void testMergeTrees() { Grouping request = new Grouping() .addLevel(new GroupingLevel() @@ -732,4 +750,5 @@ public class MergeTestCase extends TestCase { assertEquals(expect.toString(), tmp.getRoot().toString()); assertEquals(expect, tmp.getRoot()); } + } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/ExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/ExpressionTestCase.java index d76db4e8147..12a4cacaa97 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/ExpressionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/ExpressionTestCase.java @@ -5,16 +5,24 @@ import com.yahoo.io.GrowableByteBuffer; import com.yahoo.text.Utf8; import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.Identifiable; -import junit.framework.TestCase; +import org.junit.Test; import java.nio.ByteBuffer; import java.util.Arrays; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * @author baldersheim */ -public class ExpressionTestCase extends TestCase { +public class ExpressionTestCase { + + private static final double delta = 0.00000001; + @Test public void testRangeBucketPreDefFunctionNode() { assertMultiArgFunctionNode(new RangeBucketPreDefFunctionNode(new StringBucketResultNodeVector().add(new StringBucketResultNode("10", "20")), new AttributeNode("foo"))); assertEquals(new RangeBucketPreDefFunctionNode(), new RangeBucketPreDefFunctionNode()); @@ -26,6 +34,7 @@ public class ExpressionTestCase extends TestCase { new RangeBucketPreDefFunctionNode(new StringBucketResultNodeVector().add(new StringBucketResultNode("10", "20")), new AttributeNode("bar"))); } + @Test public void testFixedWidthBucketFunctionNode() { assertMultiArgFunctionNode(new FixedWidthBucketFunctionNode()); assertEquals(new FixedWidthBucketFunctionNode(), new FixedWidthBucketFunctionNode()); @@ -37,6 +46,7 @@ public class ExpressionTestCase extends TestCase { new FixedWidthBucketFunctionNode(new IntegerResultNode(5), new AttributeNode("bar"))); } + @Test public void testIntegerBucketResultNodeVector() { assertResultNode(new IntegerBucketResultNodeVector().add(new IntegerBucketResultNode(10, 20))); assertEquals(new IntegerBucketResultNodeVector().add(new IntegerBucketResultNode(10, 20)), @@ -47,6 +57,7 @@ public class ExpressionTestCase extends TestCase { new IntegerBucketResultNodeVector().add(new IntegerBucketResultNode(11, 20))); } + @Test public void testFloatBucketResultNodeVector() { assertResultNode(new FloatBucketResultNodeVector().add(new FloatBucketResultNode(10, 20))); assertEquals(new FloatBucketResultNodeVector().add(new FloatBucketResultNode(10, 20)), @@ -57,6 +68,7 @@ public class ExpressionTestCase extends TestCase { new FloatBucketResultNodeVector().add(new FloatBucketResultNode(11, 20))); } + @Test public void testStringBucketResultNodeVector() { assertResultNode(new StringBucketResultNodeVector().add(new StringBucketResultNode("10", "20"))); assertEquals(new StringBucketResultNodeVector().add(new StringBucketResultNode("10", "20")), @@ -67,6 +79,7 @@ public class ExpressionTestCase extends TestCase { new StringBucketResultNodeVector().add(new StringBucketResultNode("11", "20"))); } + @Test public void testIntegerBucketResultNode() { assertResultNode(new IntegerBucketResultNode(10, 20)); assertEquals(new IntegerBucketResultNode(10, 20), new IntegerBucketResultNode(10, 20)); @@ -74,6 +87,7 @@ public class ExpressionTestCase extends TestCase { assertNotEquals(new IntegerBucketResultNode(10, 20), new IntegerBucketResultNode(10, 21)); } + @Test public void testFloatBucketResultNode() { assertResultNode(new FloatBucketResultNode(10.0, 20.0)); assertEquals(new FloatBucketResultNode(10.0, 20.0), new FloatBucketResultNode(10.0, 20.0)); @@ -81,6 +95,7 @@ public class ExpressionTestCase extends TestCase { assertNotEquals(new FloatBucketResultNode(10.0, 20.0), new FloatBucketResultNode(10.0, 21.0)); } + @Test public void testStringBucketResultNode() { assertResultNode(new StringBucketResultNode("10.0", "20.0")); assertEquals(new StringBucketResultNode("10.0", "20.0"), new StringBucketResultNode("10.0", "20.0")); @@ -99,6 +114,7 @@ public class ExpressionTestCase extends TestCase { new StringBucketResultNode("11.0", "19.0"), new StringBucketResultNode("11.0", "20.0")); } + @Test public void testPositiveInfinity() { PositiveInfinityResultNode inf = new PositiveInfinityResultNode(); PositiveInfinityResultNode inf2 = new PositiveInfinityResultNode(); @@ -106,6 +122,7 @@ public class ExpressionTestCase extends TestCase { assertEquals(inf, inf2); } + @Test public void testAddFunctionNode() { assertMultiArgFunctionNode(new AddFunctionNode()); assertFunctionNode(new AddFunctionNode().addArg(new ConstantNode(new IntegerResultNode(2))) @@ -119,6 +136,7 @@ public class ExpressionTestCase extends TestCase { 5, 5.0, "5.0", doubleAsRaw(5.0)); } + @Test public void testAndFunctionNode() { assertMultiArgFunctionNode(new AndFunctionNode()); assertFunctionNode(new AndFunctionNode().addArg(new ConstantNode(new IntegerResultNode(3))) @@ -126,11 +144,13 @@ public class ExpressionTestCase extends TestCase { 3, 3.0, "3", longAsRaw(3)); } + @Test public void testZCurveFunctionNode() { assertMultiArgFunctionNode( new ZCurveFunctionNode(new ConstantNode(new IntegerResultNode(7)), ZCurveFunctionNode.Dimension.Y)); } + @Test public void testTimeStampFunctionNode() { assertMultiArgFunctionNode(new TimeStampFunctionNode(new AttributeNode("testattribute"), TimeStampFunctionNode.TimePart.Hour, true)); assertEquals(new TimeStampFunctionNode(new AttributeNode("testattribute"), TimeStampFunctionNode.TimePart.Hour, true), @@ -152,11 +172,13 @@ public class ExpressionTestCase extends TestCase { false)); } + @Test public void testExpressionRefNode() { AggregationRefNode ref = new AggregationRefNode(3); assertEquals(3, ref.getIndex()); } + @Test public void testAttributeNode() { try { new AttributeNode(null); @@ -192,6 +214,7 @@ public class ExpressionTestCase extends TestCase { assertFalse(b.equals(c)); } + @Test public void testInterpolatedLookupNode() { ExpressionNode argA = new ConstantNode(new FloatResultNode(2.71828182846)); ExpressionNode argB = new ConstantNode(new FloatResultNode(3.14159265359)); @@ -236,6 +259,7 @@ public class ExpressionTestCase extends TestCase { assertFalse(a1.equals(a2)); } + @Test public void testCatFunctionNode() { assertMultiArgFunctionNode(new CatFunctionNode()); assertFunctionNode(new CatFunctionNode().addArg(new ConstantNode(new RawResultNode(asRaw('1', '2')))) @@ -243,6 +267,7 @@ public class ExpressionTestCase extends TestCase { 0, 0.0, "1234", asRaw('1', '2', '3', '4')); } + @Test public void testStrCatFunctionNode() { assertMultiArgFunctionNode(new StrCatFunctionNode()); assertFunctionNode(new StrCatFunctionNode().addArg(new ConstantNode(new StringResultNode("foo"))) @@ -250,6 +275,7 @@ public class ExpressionTestCase extends TestCase { 0, 0.0, "foobar", stringAsRaw("foobar")); } + @Test public void testDivideFunctionNode() { assertMultiArgFunctionNode(new DivideFunctionNode()); assertFunctionNode(new DivideFunctionNode().addArg(new ConstantNode(new IntegerResultNode(10))) @@ -263,6 +289,7 @@ public class ExpressionTestCase extends TestCase { 1, 0.5, "0.5", doubleAsRaw(0.5)); } + @Test public void testDocumentFieldNode() { try { new DocumentFieldNode(null); @@ -298,10 +325,11 @@ public class ExpressionTestCase extends TestCase { assertFalse(b.equals(c)); } + @Test public void testFloatResultNode() { FloatResultNode a = new FloatResultNode(7.3); assertEquals(a.getInteger(), 7); - assertEquals(a.getFloat(), 7.3); + assertEquals(a.getFloat(), 7.3, delta); assertEquals(a.getString(), "7.3"); assertEquals(a.getNumber(), new Double(7.3)); byte[] raw = a.getRaw(); @@ -313,11 +341,12 @@ public class ExpressionTestCase extends TestCase { FloatResultNode b = new FloatResultNode(7.5); assertEquals(b.getInteger(), 8); - assertEquals(b.getFloat(), 7.5); + assertEquals(b.getFloat(), 7.5, delta); assertEquals(b.getString(), "7.5"); assertEquals(b.getNumber(), new Double(7.5)); } + @Test public void testGetDocIdNamespaceSpecificFunctionNode() { GetDocIdNamespaceSpecificFunctionNode a = new GetDocIdNamespaceSpecificFunctionNode(new IntegerResultNode(7)); assertTrue(a.getResult() instanceof IntegerResultNode); @@ -340,6 +369,7 @@ public class ExpressionTestCase extends TestCase { } } + @Test public void testGetYMUMChecksumFunctionNode() { GetYMUMChecksumFunctionNode a = new GetYMUMChecksumFunctionNode(); assertTrue(a.getResult() instanceof IntegerResultNode); @@ -358,10 +388,11 @@ public class ExpressionTestCase extends TestCase { } } + @Test public void testIntegerResultNode() { IntegerResultNode a = new IntegerResultNode(7); assertEquals(a.getInteger(), 7); - assertEquals(a.getFloat(), 7.0); + assertEquals(a.getFloat(), 7.0, delta); assertEquals(a.getString(), "7"); assertEquals(a.getNumber(), new Long(7)); byte[] raw = a.getRaw(); @@ -371,6 +402,7 @@ public class ExpressionTestCase extends TestCase { compare(new IntegerResultNode(-1), new IntegerResultNode(0), new IntegerResultNode(0x80000000L)); } + @Test public void testMaxFunctionNode() { assertMultiArgFunctionNode(new MaxFunctionNode()); assertFunctionNode(new MaxFunctionNode().addArg(new ConstantNode(new IntegerResultNode(3))) @@ -381,6 +413,7 @@ public class ExpressionTestCase extends TestCase { 5, 5.0, "5.0", doubleAsRaw(5.0)); } + @Test public void testMD5BitFunctionNode() { try { new MD5BitFunctionNode(null, 64); @@ -403,6 +436,7 @@ public class ExpressionTestCase extends TestCase { assertUnaryBitFunctionNode(new MD5BitFunctionNode()); } + @Test public void testMinFunctionNode() { assertMultiArgFunctionNode(new MinFunctionNode()); assertFunctionNode(new MinFunctionNode().addArg(new ConstantNode(new IntegerResultNode(3))) @@ -413,6 +447,7 @@ public class ExpressionTestCase extends TestCase { 5, 4.9999999, "4.9999999", doubleAsRaw(4.9999999)); } + @Test public void testModuloFunctionNode() { assertMultiArgFunctionNode(new ModuloFunctionNode()); assertFunctionNode(new ModuloFunctionNode().addArg(new ConstantNode(new IntegerResultNode(13))) @@ -423,6 +458,7 @@ public class ExpressionTestCase extends TestCase { 5, 4.9999999, "4.9999999", doubleAsRaw(4.9999999)); } + @Test public void testMultiplyFunctionNode() { assertMultiArgFunctionNode(new MultiplyFunctionNode()); assertFunctionNode(new MultiplyFunctionNode().addArg(new ConstantNode(new IntegerResultNode(3))) @@ -433,6 +469,7 @@ public class ExpressionTestCase extends TestCase { 23, 22.5, "22.5", doubleAsRaw(22.5)); } + @Test public void testNegateFunctionNode() { assertMultiArgFunctionNode(new NegateFunctionNode()); assertFunctionNode(new NegateFunctionNode().addArg(new ConstantNode(new IntegerResultNode(3))), @@ -441,73 +478,80 @@ public class ExpressionTestCase extends TestCase { -3, -3.0, "-3.0", doubleAsRaw(-3.0)); } + @Test public void testSortFunctionNode() { assertMultiArgFunctionNode(new SortFunctionNode()); - } + @Test public void testReverseFunctionNode() { assertMultiArgFunctionNode(new ReverseFunctionNode()); } + @Test public void testToIntFunctionNode() { assertMultiArgFunctionNode(new ToIntFunctionNode()); assertFunctionNode(new ToIntFunctionNode().addArg(new ConstantNode(new StringResultNode("1337"))), 1337, 1337.0, "1337", longAsRaw(1337)); } + @Test public void testToFloatFunctionNode() { assertMultiArgFunctionNode(new ToFloatFunctionNode()); assertFunctionNode(new ToFloatFunctionNode().addArg(new ConstantNode(new FloatResultNode(3.14))), 3, 3.14, "3.14", doubleAsRaw(3.14)); } + @Test public void testMathFunctionNode() { assertMultiArgFunctionNode(new MathFunctionNode(MathFunctionNode.Function.LOG10)); assertFunctionNode(new MathFunctionNode(MathFunctionNode.Function.LOG10).addArg(new ConstantNode(new IntegerResultNode(100000))), 5, 5.0, "5.0", doubleAsRaw(5.0)); } + @Test public void testStrLenFunctionNode() { assertMultiArgFunctionNode(new StrLenFunctionNode()); assertFunctionNode(new StrLenFunctionNode().addArg(new ConstantNode(new StringResultNode("foo"))), 3, 3.0, "3", longAsRaw(3)); } + @Test public void testNormalizeSubjectFunctionNode() { assertMultiArgFunctionNode(new NormalizeSubjectFunctionNode()); assertFunctionNode(new NormalizeSubjectFunctionNode().addArg(new ConstantNode(new StringResultNode("Re: Your mail"))), 0, 0, "Your mail", stringAsRaw("Your mail")); } + @Test public void testNormalizeSubjectFunctionNode2() { assertMultiArgFunctionNode(new NormalizeSubjectFunctionNode()); assertFunctionNode(new NormalizeSubjectFunctionNode().addArg(new ConstantNode(new StringResultNode("Your mail"))), 0, 0, "Your mail", stringAsRaw("Your mail")); } + @Test public void testNumElemFunctionNode() { assertMultiArgFunctionNode(new NumElemFunctionNode()); assertFunctionNode(new NumElemFunctionNode().addArg(new ConstantNode(new IntegerResultNode(1337))), 1, 1.0, "1", longAsRaw(1)); } + @Test public void testToStringFunctionNode() { assertMultiArgFunctionNode(new ToStringFunctionNode()); assertFunctionNode(new ToStringFunctionNode().addArg(new ConstantNode(new IntegerResultNode(1337))), 1337, 1337.0, "1337", stringAsRaw("1337")); } + @Test public void testToRawFunctionNode() { assertMultiArgFunctionNode(new ToRawFunctionNode()); assertFunctionNode(new ToRawFunctionNode().addArg(new ConstantNode(new IntegerResultNode(1337))), 1337, 1337.0, "1337", longAsRaw(1337)); } - public void testNullResultNode() { - // TODO: Implement. - } - + @Test public void testOrFunctionNode() { assertMultiArgFunctionNode(new OrFunctionNode()); assertFunctionNode(new OrFunctionNode().addArg(new ConstantNode(new IntegerResultNode(2))) @@ -515,6 +559,7 @@ public class ExpressionTestCase extends TestCase { 6, 6.0, "6", longAsRaw(6)); } + @Test public void testDebugWaitFunctionNode() { assertFunctionNode( new DebugWaitFunctionNode(new OrFunctionNode().addArg(new ConstantNode(new IntegerResultNode(2))) @@ -543,6 +588,7 @@ public class ExpressionTestCase extends TestCase { assertTrue(end - start > 450); } + @Test public void testRawResultNode() { try { new RawResultNode(null); @@ -564,7 +610,7 @@ public class ExpressionTestCase extends TestCase { assertEquals(raw[1], '.'); assertEquals(raw[2], '4'); assertEquals(a.getInteger(), 0); - assertEquals(a.getFloat(), 0.0); + assertEquals(a.getFloat(), 0.0, delta); assertEquals(a.getString(), "7.4"); assertResultNode(a); compare(new RawResultNode(), new RawResultNode(new byte [] {'z'}), new RawResultNode(new byte [] {'z', 'z'})); @@ -647,6 +693,7 @@ public class ExpressionTestCase extends TestCase { assertEquals(0, large.compareTo(large)); } + @Test public void testStringResultNode() { try { new StringResultNode(null); @@ -662,7 +709,7 @@ public class ExpressionTestCase extends TestCase { } StringResultNode a = new StringResultNode("7.3"); assertEquals(a.getInteger(), 0); - assertEquals(a.getFloat(), 7.3); + assertEquals(a.getFloat(), 7.3, delta); assertEquals(a.getString(), "7.3"); byte[] raw = a.getRaw(); assertEquals(raw.length, 3); @@ -672,6 +719,7 @@ public class ExpressionTestCase extends TestCase { compare(new StringResultNode("a"), new StringResultNode("zz"), new PositiveInfinityResultNode()); } + @Test public void testXorBitFunctionNode() { try { new XorBitFunctionNode(null, 64); @@ -694,6 +742,7 @@ public class ExpressionTestCase extends TestCase { assertUnaryBitFunctionNode(new XorBitFunctionNode()); } + @Test public void testUcaFunctionNode() { try { new UcaFunctionNode(null, "foo"); @@ -716,6 +765,7 @@ public class ExpressionTestCase extends TestCase { assertUcaFunctionNode(new UcaFunctionNode(new ConstantNode(new IntegerResultNode(1337)), "foo", "bar")); } + @Test public void testNestedFunctions() { assertFunctionNode(new AddFunctionNode() .addArg(new MultiplyFunctionNode().addArg(new ConstantNode(new IntegerResultNode(3))) @@ -725,6 +775,7 @@ public class ExpressionTestCase extends TestCase { 14, 14.0, "14.0", doubleAsRaw(14.0)); } + @Test public void testArithmeticNodes() { ExpressionNode i1 = new ConstantNode(new IntegerResultNode(1)); ExpressionNode i2 = new ConstantNode(new IntegerResultNode(2)); @@ -761,6 +812,7 @@ public class ExpressionTestCase extends TestCase { assertTrue(add4.getResult() instanceof IntegerResultNode); } + @Test public void testArithmeticOperations() { ExpressionNode i1 = new ConstantNode(new IntegerResultNode(1793253241)); ExpressionNode i2 = new ConstantNode(new IntegerResultNode(1676521321)); @@ -860,14 +912,14 @@ public class ExpressionTestCase extends TestCase { node.prepare(); node.execute(); assertEquals(lexpected, node.getResult().getInteger()); - assertEquals(dexpected, node.getResult().getFloat()); + assertEquals(dexpected, node.getResult().getFloat(), delta); } public void assertFunctionNode(FunctionNode node, long lexpected, double dexpected, String sexpected, byte[] rexpected) { node.prepare(); node.execute(); assertEquals(lexpected, node.getResult().getInteger()); - assertEquals(dexpected, node.getResult().getFloat()); + assertEquals(dexpected, node.getResult().getFloat(), delta); assertEquals(sexpected, node.getResult().getString()); assertTrue(Arrays.equals(rexpected, node.getResult().getRaw())); } @@ -882,7 +934,7 @@ public class ExpressionTestCase extends TestCase { buf.flip(); node.deserialize(buf); assertEquals(oldInteger, node.getInteger()); - assertEquals(oldFloat, node.getFloat()); + assertEquals(oldFloat, node.getFloat(), delta); assertEquals(oldString, node.getString()); assertEquals(oldRaw.length, node.getRaw().length); @@ -891,7 +943,7 @@ public class ExpressionTestCase extends TestCase { buf.flip(); node.deserializeWithId(buf); assertEquals(oldInteger, node.getInteger()); - assertEquals(oldFloat, node.getFloat()); + assertEquals(oldFloat, node.getFloat(), delta); assertEquals(oldString, node.getString()); assertEquals(oldRaw.length, node.getRaw().length); @@ -900,7 +952,7 @@ public class ExpressionTestCase extends TestCase { buf.flip(); ResultNode obj = (ResultNode)Identifiable.create(buf); assertEquals(oldInteger, obj.getInteger()); - assertEquals(oldFloat, obj.getFloat()); + assertEquals(oldFloat, obj.getFloat(), delta); assertEquals(oldString, obj.getString()); assertEquals(oldRaw.length, obj.getRaw().length); @@ -913,7 +965,7 @@ public class ExpressionTestCase extends TestCase { buf.flip(); Identifiable created = Identifiable.create(buf); assertEquals(node, created); - assertEquals(buf.getBuf().hasRemaining(), false); + assertFalse(buf.getBuf().hasRemaining()); Identifiable cloned = created.clone(); assertEquals(node, cloned); BufferSerializer createdBuffer = new BufferSerializer(new GrowableByteBuffer()); @@ -929,4 +981,5 @@ public class ExpressionTestCase extends TestCase { } return created; } + } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/ForceLoadTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/ForceLoadTestCase.java index a5d216ff089..fa2c9cf1b1e 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/ForceLoadTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/ForceLoadTestCase.java @@ -1,19 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.expression; -public class ForceLoadTestCase extends junit.framework.TestCase { +import org.junit.Test; - public ForceLoadTestCase(String name) { - super(name); - } +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class ForceLoadTestCase { + @Test public void testLoadClasses() { try { new com.yahoo.searchlib.expression.ForceLoad(); assertTrue(com.yahoo.searchlib.expression.ForceLoad.forceLoad()); } catch (com.yahoo.system.ForceLoadError e) { e.printStackTrace(); - assertTrue(false); + fail("Load failed"); } } + } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/expression/ObjectVisitorTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/expression/ObjectVisitorTestCase.java index 1d1fa8251fd..286a6a702db 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/expression/ObjectVisitorTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/expression/ObjectVisitorTestCase.java @@ -2,18 +2,18 @@ package com.yahoo.searchlib.expression; import com.yahoo.vespa.objects.ObjectDumper; -import com.yahoo.searchlib.expression.FixedWidthBucketFunctionNode; -import com.yahoo.searchlib.expression.IntegerResultNode; -import com.yahoo.searchlib.expression.AttributeNode; -import junit.framework.TestCase; +import org.junit.Test; import java.util.Arrays; +import static org.junit.Assert.assertEquals; + /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ -public class ObjectVisitorTestCase extends TestCase { +public class ObjectVisitorTestCase { + @Test public void testObjectDumper() { assertDump("test: <NULL>\n", null); assertDump("test: 1\n", 1); @@ -58,4 +58,5 @@ public class ObjectVisitorTestCase extends TestCase { dump.visit("test", obj); assertEquals(expected, dump.toString()); } + } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/SemanticDistanceTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/SemanticDistanceTestCase.java index 68db4867cf7..eb0e46ad6dc 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/SemanticDistanceTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/SemanticDistanceTestCase.java @@ -1,11 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchlib.ranking.features.fieldmatch; -import com.yahoo.searchlib.ranking.features.fieldmatch.FieldMatchMetricsComputer; +import org.junit.Before; +import org.junit.Test; import java.util.Set; import java.util.HashSet; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + /** * The "semantic distance" refers to the non-continuous distance from a token * to the next token used by the string match metrics algorithm. This class @@ -14,24 +18,21 @@ import java.util.HashSet; * * @author bratseth */ -public class SemanticDistanceTestCase extends junit.framework.TestCase { +public class SemanticDistanceTestCase { FieldMatchMetricsComputer c; - public SemanticDistanceTestCase(String name) { - super(name); - } - - @Override + @Before public void setUp() { c=new FieldMatchMetricsComputer(); - StringBuilder field=new StringBuilder(); - for (int i=0; i<150; i++) + StringBuilder field = new StringBuilder(); + for (int i = 0; i < 150; i++) field.append("t" + i + " "); - c.compute("query",field.toString()); // Just to set the field value + c.compute("query", field.toString()); // Just to set the field value } /** Must be true using any semantic distance function */ + @Test public void testBothWayConversionProducesOriginalValue() { assertBothWayConversionProducesOriginalValue(50); assertBothWayConversionProducesOriginalValue(10); @@ -43,6 +44,7 @@ public class SemanticDistanceTestCase extends junit.framework.TestCase { } /** Must be true using any semantic distance function */ + @Test public void testFunctionsAreOneToOne() { assertFunctionsAreOneToOne(50); assertFunctionsAreOneToOne(10); @@ -54,52 +56,54 @@ public class SemanticDistanceTestCase extends junit.framework.TestCase { } /** Specific to this particular distance function */ + @Test public void testFunction() { - int zeroJ=50; - assertFunction(50,0,zeroJ); - assertFunction(59,9,zeroJ); - assertFunction(49,10,zeroJ); - assertFunction(40,19,zeroJ); - assertFunction(60,20,zeroJ); - assertFunction(149,109,zeroJ); - assertFunction(39,110,zeroJ); - assertFunction(0,149,zeroJ); - - zeroJ=0; - assertFunction(0,0,zeroJ); - assertFunction(10,10,zeroJ); - assertFunction(20,20,zeroJ); - assertFunction(149,149,zeroJ); - - zeroJ=5; - assertFunction(5,0,zeroJ); - assertFunction(10,5,zeroJ); - assertFunction(14,9,zeroJ); - assertFunction(4,10,zeroJ); - assertFunction(0,14,zeroJ); - assertFunction(15,15,zeroJ); - assertFunction(25,25,zeroJ); - assertFunction(149,149,zeroJ); - - zeroJ=149; - assertFunction(149,0,zeroJ); - assertFunction(140,9,zeroJ); - assertFunction(130,19,zeroJ); - assertFunction(0,149,zeroJ); - - zeroJ=145; - assertFunction(145,0,zeroJ); - assertFunction(149,4,zeroJ); - assertFunction(144,5,zeroJ); - assertFunction(140,9,zeroJ); - assertFunction(135,14,zeroJ); - assertFunction(125,24,zeroJ); - assertFunction(0,149,zeroJ); + int zeroJ = 50; + assertFunction(50,0, zeroJ); + assertFunction(59,9, zeroJ); + assertFunction(49,10, zeroJ); + assertFunction(40,19, zeroJ); + assertFunction(60,20, zeroJ); + assertFunction(149,109, zeroJ); + assertFunction(39,110, zeroJ); + assertFunction(0,149, zeroJ); + + zeroJ = 0; + assertFunction(0,0, zeroJ); + assertFunction(10,10, zeroJ); + assertFunction(20,20, zeroJ); + assertFunction(149,149, zeroJ); + + zeroJ = 5; + assertFunction(5,0, zeroJ); + assertFunction(10,5, zeroJ); + assertFunction(14,9, zeroJ); + assertFunction(4,10, zeroJ); + assertFunction(0,14, zeroJ); + assertFunction(15,15, zeroJ); + assertFunction(25,25, zeroJ); + assertFunction(149,149, zeroJ); + + zeroJ = 149; + assertFunction(149,0, zeroJ); + assertFunction(140,9, zeroJ); + assertFunction(130,19, zeroJ); + assertFunction(0,149, zeroJ); + + zeroJ = 145; + assertFunction(145,0, zeroJ); + assertFunction(149,4, zeroJ); + assertFunction(144,5, zeroJ); + assertFunction(140,9, zeroJ); + assertFunction(135,14, zeroJ); + assertFunction(125,24, zeroJ); + assertFunction(0,149, zeroJ); } /** Hits both limits at once */ + @Test public void testSmallField() { - c=new FieldMatchMetricsComputer(); + c = new FieldMatchMetricsComputer(); c.compute("query","my field value four"); // Just to set the field value assertBothWayConversionProducesOriginalValue(2); assertBothWayConversionProducesOriginalValue(0); @@ -109,33 +113,34 @@ public class SemanticDistanceTestCase extends junit.framework.TestCase { assertFunctionsAreOneToOne(3); int zeroJ=2; - assertFunction(2,0,zeroJ); - assertFunction(3,1,zeroJ); - assertFunction(1,2,zeroJ); - assertFunction(0,3,zeroJ); + assertFunction(2,0, zeroJ); + assertFunction(3,1, zeroJ); + assertFunction(1,2, zeroJ); + assertFunction(0,3, zeroJ); } private void assertBothWayConversionProducesOriginalValue(int zeroJ) { // Starting point in the middle - for (int j=0; j<c.getField().terms().size(); j++) { - int semanticDistance=c.fieldIndexToSemanticDistance(j,zeroJ); + for (int j = 0; j < c.getField().terms().size(); j++) { + int semanticDistance=c.fieldIndexToSemanticDistance(j, zeroJ); assertTrue("Using zeroJ=" + zeroJ + ": " + semanticDistance +">=0", semanticDistance >= 0); - int backConvertedJ=c.semanticDistanceToFieldIndex(semanticDistance,zeroJ); + int backConvertedJ=c.semanticDistanceToFieldIndex(semanticDistance, zeroJ); assertEquals("Using zeroJ=" + zeroJ + ": " + j + "->" + semanticDistance + "->" + backConvertedJ,j, backConvertedJ); } } private void assertFunctionsAreOneToOne(int zeroJ) { - Set<Integer> distances=new HashSet<Integer>(); - for (int j=0; j<c.getField().terms().size(); j++) { - int semanticDistance=c.fieldIndexToSemanticDistance(j,zeroJ); - assertTrue("Using zeroJ=" + zeroJ + ": " + j + "->" + semanticDistance + " is unique", ! distances.contains(semanticDistance)); + Set<Integer> distances = new HashSet<>(); + for (int j = 0; j < c.getField().terms().size(); j++) { + int semanticDistance = c.fieldIndexToSemanticDistance(j,zeroJ); + assertTrue("Using zeroJ=" + zeroJ + ": " + j + "->" + semanticDistance + " is unique", + ! distances.contains(semanticDistance)); distances.add(semanticDistance); } } - private void assertFunction(int j,int semanticDistance,int zeroJ) { - assertEquals(j + "->" + semanticDistance,semanticDistance,c.fieldIndexToSemanticDistance(j,zeroJ)); + private void assertFunction(int j,int semanticDistance, int zeroJ) { + assertEquals(j + "->" + semanticDistance,semanticDistance, c.fieldIndexToSemanticDistance(j,zeroJ)); } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/reference/test/OptimalStringAlignmentTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/reference/test/OptimalStringAlignmentTestCase.java index b2db0d0f805..c9b812894ba 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/reference/test/OptimalStringAlignmentTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/reference/test/OptimalStringAlignmentTestCase.java @@ -2,20 +2,22 @@ package com.yahoo.searchlib.ranking.features.fieldmatch.reference.test; import com.yahoo.searchlib.ranking.features.fieldmatch.reference.OptimalStringAlignmentDistance; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; /** * @author bratseth */ -public class OptimalStringAlignmentTestCase extends junit.framework.TestCase { +public class OptimalStringAlignmentTestCase { - public OptimalStringAlignmentTestCase(String name) { - super(name); - } + private final double delta = 0.0000000001; + @Test public void testEditDistance() { // Edit distance, substitution, deletion, insertion, transposition, query, field, print? - boolean print=false; + boolean print = false; assertEditDistance(0,0,0,0,0,"niels bohr","niels bohr",print); assertEditDistance(1,1,0,0,0,"niels","bohr",print); assertEditDistance(1,0,0,1,0,"niels","niels bohr",print); @@ -27,8 +29,9 @@ public class OptimalStringAlignmentTestCase extends junit.framework.TestCase { assertEditDistance(3,2,0,1,0,"niels bohr i kopenhagen","niels henrik bor i stockholm",print); } + @Test public void testEditDistanceAsRelevance() { - boolean print=false; + boolean print = false; assertEditDistance(2,0,0,2,0,"niels bohr","niels blah blah bohr",print); assertEditDistance(4,0,1,3,0,"niels bohr","bohr blah blah niels",print); // Not desired assertEditDistance(4,2,0,2,0,"niels bohr","koko blah blah bahia",print); @@ -48,11 +51,11 @@ public class OptimalStringAlignmentTestCase extends junit.framework.TestCase { System.out.println(); } - assertEquals("Substitutions",(float)substitution,e.getSubstitutions()); - assertEquals("Deletions",(float)deletion,e.getDeletions()); - assertEquals("Insertions",(float)insertion,e.getInsertions()); - assertEquals("Transpositions",(float)transposition,e.getTranspositions()); - assertEquals("Total",(float)total,e.getTotal()); + assertEquals("Substitutions",(float)substitution,e.getSubstitutions(), delta); + assertEquals("Deletions",(float)deletion,e.getDeletions(), delta); + assertEquals("Insertions",(float)insertion,e.getInsertions(), delta); + assertEquals("Transpositions",(float)transposition,e.getTranspositions(), delta); + assertEquals("Total",(float)total,e.getTotal(), delta); } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/test/FieldMatchMetricsTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/test/FieldMatchMetricsTestCase.java index 6f9b0f03b6b..4918ee5bdcd 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/test/FieldMatchMetricsTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/ranking/features/fieldmatch/test/FieldMatchMetricsTestCase.java @@ -8,21 +8,21 @@ import com.yahoo.searchlib.ranking.features.fieldmatch.FieldMatchMetricsComputer import com.yahoo.searchlib.ranking.features.fieldmatch.FieldMatchMetricsParameters; import com.yahoo.searchlib.ranking.features.fieldmatch.QueryTerm; import com.yahoo.searchlib.ranking.features.fieldmatch.Query; +import org.junit.Test; import java.util.List; +import static org.junit.Assert.assertEquals; + /** * Tests of calculation of all the string match metrics. * Add true as the fourth parameter to assertMetrics to see a trace of what the test is doing. * * @author bratseth */ -public class FieldMatchMetricsTestCase extends junit.framework.TestCase { - - public FieldMatchMetricsTestCase(String name) { - super(name); - } +public class FieldMatchMetricsTestCase { + @Test public void testOutOfOrder() { assertMetrics("outOfOrder:0","a","a"); assertMetrics("outOfOrder:0","a b c","a b c"); @@ -33,6 +33,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("outOfOrder:2", "a b c d e", "c x a b x x x x x e x x d"); } + @Test public void testSegments() { assertMetrics("segments:1","a","a"); assertMetrics("segments:1","a b c","a b c"); @@ -45,6 +46,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("segments:2 gaps:1","a y y b c","x x x b x x c x x x x x x x x x x x x x x x x x x x a x x"); } + @Test public void testGaps() { assertMetrics("gaps:0","a","a"); assertMetrics("gaps:0","x�a","a"); @@ -59,6 +61,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("gaps:0","y a b c","a b c x"); } + @Test public void testHead() { assertMetrics("head:0","a","a"); assertMetrics("head:0","y","a"); @@ -68,6 +71,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("head:2", "a b c", "x x c x x x x x x x x x x x x x x x a b"); } + @Test public void testTail() { assertMetrics("tail:0","a","a"); assertMetrics("tail:0","y","a"); @@ -77,6 +81,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("tail:0","a b c","x x c x x x x x x x x x x x x x x x a b"); } + @Test public void testLongestSequence() { assertMetrics("longestSequence:1","a","a"); assertMetrics("longestSequence:1","a","a b c"); @@ -89,6 +94,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("longestSequence:4 segments:1","a b c d","x x a b x x x x x x x x x x x x x x x x x c d x x a b c d"); } + @Test public void testMatches() { assertMetrics("matches:1 queryCompleteness:1 fieldCompleteness:1", "a","a"); assertMetrics("matches:3 queryCompleteness:1 fieldCompleteness:1", "a b c","a b c"); @@ -96,6 +102,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("matches:3 queryCompleteness:0.5 fieldCompleteness:0.25","a y y b c y","a x x b c x a x a b x x"); } + @Test public void testCompleteness() { assertMetrics("completeness:1 queryCompleteness:1 fieldCompleteness:1", "a","a"); assertMetrics("completeness:0 queryCompleteness:0 fieldCompleteness:0", "a","x"); @@ -109,6 +116,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("completeness:0.97 queryCompleteness:1 fieldCompleteness:0.4","a b","a b b b b"); } + @Test public void testOrderness() { assertMetrics("orderness:1", "a","a"); assertMetrics("orderness:1", "a","x"); @@ -120,6 +128,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("orderness:1", "a b","b x x x x x x x x x x x x x x x x x x x x x a"); } + @Test public void testRelatedness() { assertMetrics("relatedness:1", "a","a"); assertMetrics("relatedness:0", "a","x"); @@ -129,6 +138,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("relatedness:0.5","a y b y y y c","a b x x x x x x x x x x x x x x x x x x x x x x x c"); } + @Test public void testLongestSequenceRatio() { assertMetrics("longestSequenceRatio:1", "a","a"); assertMetrics("longestSequenceRatio:0", "a","x"); @@ -140,6 +150,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("longestSequenceRatio:0.75","a b c d","x x a b x a x c d a b c x d x"); } + @Test public void testEarliness() { assertMetrics("earliness:1", "a","a"); assertMetrics("earliness:0", "a","x"); @@ -151,6 +162,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("earliness:0.2", "a b c","x b c a x x x x a x x x x x x x a b c x x"); } + @Test public void testWeight() { assertMetrics("weight:1", "a","a"); assertMetrics("weight:0", "y","a"); @@ -216,6 +228,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { } /** Calculated the same way as weight */ + @Test public void testSignificance() { assertMetrics("significance:1", "a","a"); assertMetrics("significance:0", "a","x"); @@ -280,6 +293,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("significance:0","a%0 b%0","",0.3f); } + @Test public void testImportance() { assertMetrics("importance:0.75","a b c", "a x x b x c c c",600); assertMetrics("importance:0.85","a b!500 c","a x x b x c c c",1000); @@ -290,6 +304,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("importance:0.85","a b!500%0.5 c","a x x b x c c c",1000); } + @Test public void testOccurrence() { assertMetrics("occurrence:0","a","x"); assertMetrics("occurrence:1","a","a"); @@ -313,6 +328,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("occurrence:1", "a b","a a a a a a a a a a a b b b b b b b b b b b",false,c); // Field is too large to consider field length } + @Test public void testAbsoluteOccurrence() { assertMetrics("absoluteOccurrence:0", "a","x"); assertMetrics("absoluteOccurrence:0.01","a","a"); @@ -336,6 +352,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("absoluteOccurrence:1", "a b","a a a a a a a a a a a b b b b b b b b b b b",false,c); // Field is too large to consider field length } + @Test public void testWeightedOccurrence() { assertMetrics("weightedOccurrence:0","a!200","x"); assertMetrics("weightedOccurrence:1","a!200","a"); @@ -378,6 +395,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("weightedOccurrence:1", "a!200 b","a a a a a a a a a a a b b b b b b b b b b b",false,c); // Field is too large to consider field length } + @Test public void testWeightedAbsoluteOccurrence() { assertMetrics("weightedAbsoluteOccurrence:0", "a!200","x"); assertMetrics("weightedAbsoluteOccurrence:0.01", "a!200","a"); @@ -420,6 +438,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("weightedAbsoluteOccurrence:1", "a!200 b","a a a a a a a a a a a b b b b b b b b b b b",false,c); // Field is too large to consider field length } + @Test public void testSignificantOccurrence() { assertMetrics("significantOccurrence:0","a%0.2","x"); assertMetrics("significantOccurrence:1","a%0.2","a"); @@ -462,6 +481,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("significantOccurrence:1", "a%0.2 b","a a a a a a a a a a a b b b b b b b b b b b",false,c); // Field is too large to consider field length } + @Test public void testUnweightedProximity() { assertMetrics("unweightedProximity:1", "a","a"); assertMetrics("unweightedProximity:1", "a b c","a b c"); @@ -476,6 +496,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("unweightedProximity:0.5", "y a b c","a x x b x x c x"); } + @Test public void testReverseProximity() { assertMetrics("unweightedProximity:0.33", "a b","b a"); assertMetrics("unweightedProximity:0.62", "a b c","c a b"); @@ -485,6 +506,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("unweightedProximity:0.9275","a b c d e","a b x c d e"); } + @Test public void testProximity() { assertMetrics("absoluteProximity:0.1 proximity:1", "a b","a b"); assertMetrics("absoluteProximity:0.3 proximity:1", "a 0.3:b","a b"); @@ -509,6 +531,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { * Tests exactness (using field exactness only - nothing additional of interest to test with query exactness * as that is just another number multiplied with the term exactness) */ + @Test public void testExactness() { assertMetrics("exactness:1", "a b c","a x b x x c"); assertMetrics("exactness:0.9", "a b c","a x b:0.7 x x c"); @@ -517,12 +540,14 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("exactness:0.65", "a b c!200","a x b:0.6 x x c:0.5"); } + @Test public void testMultiSegmentProximity() { assertMetrics("absoluteProximity:0.1 proximity:1", "a b c", "a b x x x x x x x x x x x x x x x x x x x x x x c"); assertMetrics("absoluteProximity:0.05 proximity:0.5","a b c", "a x x b x x x x x x x x x x x x x x x x x x x x x x c"); assertMetrics("absoluteProximity:0.075 proximity:0.75","a b c d","a x x b x x x x x x x x x x x x x x x x x x x x x x c d"); } + @Test public void testSegmentDistance() { assertMetrics("segmentDistance:13 absoluteProximity:0.1", "a b c","a b x x x x x x x x x x c"); assertMetrics("segmentDistance:13 absoluteProximity:0.5", "a 0.5:b c","a b x x x x x x x x x x c"); @@ -534,6 +559,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("segmentDistance:25 absoluteProximity:0.1", "a b c","c x x x x x x x x x x x b x x x x x x x x x x a"); } + @Test public void testSegmentProximity() { assertMetrics("segmentProximity:1", "a","a"); assertMetrics("segmentProximity:0", "a","x"); @@ -546,6 +572,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { } /** Test cases where we choose between multiple different segmentations */ + @Test public void testSegmentSelection() { assertMetrics("segments:2 absoluteProximity:0.1 proximity:1 segmentStarts:19,41", "a b c d e","x a b x c x x x x x x x x x x x x x x a b c x x x x x x x x x e x d x c d x x x c d e"); @@ -567,12 +594,14 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("segments:1 segmentStarts:0","a b c d","a b x x x x x x x x c d x x x x x x x x x x x a b x x x x x x x x x x x c d"); } + @Test public void testMoreThanASegmentLengthOfUnmatchedQuery() { assertMetrics("absoluteProximity:0.1 proximity:1","a b y y y y y y y y y y y y y y y","a b"); assertMetrics("segments:2 absoluteProximity:0.1 proximity:1","a b c d y y y y y y y y y y y y y y y","a b x x x x x x x x x x x x x x x x x x c d"); assertMetrics("segments:2 absoluteProximity:0.1 proximity:1","a b y y y y y y y y y y y y y y y c d","a b x x x x x x x x x x x x x x x x x x c d"); } + @Test public void testQueryRepeats() { // Not really handled perfectly, but good enough assertMetrics("absoluteProximity:0.1 proximity:1 head:0 tail:0", "a a a","a"); @@ -588,6 +617,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("matches:2 fieldCompleteness:1","a b b b","a b"); } + @Test public void testZeroCases() { assertMetrics("absoluteProximity:0.1 proximity:1 matches:0 exactness:0","y","a"); assertMetrics("absoluteProximity:0.1 proximity:1 matches:0 exactness:0","a","x"); @@ -596,8 +626,8 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("absoluteProximity:0.1 proximity:1 matches:0 exactness:0","",""); } + @Test public void testExceedingIterationLimit() { - { // Segments found: a x x b and c d FieldMatchMetricsParameters p=new FieldMatchMetricsParameters(); p.setMaxAlternativeSegmentations(0); @@ -620,6 +650,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { } } + @Test public void testMatch() { // Ordered by decreasing match score per query assertMetrics("match:1", "a","a"); @@ -647,6 +678,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { assertMetrics("match:0.2927","a b!200 c","x x a x b:0.7 x x x x x x x x x x x x x x x x x x x x x x"); } + @Test public void testRepeatedMatch() { // gap==1 caused by finding two possible segments due to repeated matching assertMetrics("fieldCompleteness:1 queryCompleteness:0.6667 segments:1 earliness:1 gaps:1", @@ -654,6 +686,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { } /** Three segments - improving the score on the first should impact the last */ + @Test public void testNestedAlternatives() { assertMetrics("segmentStarts:6,19,32 proximity:1", "a b c d e f", @@ -664,6 +697,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { } /** Nice demonstration of the limitations of this algorithm: Segment end points are determined greedily */ + @Test public void testSegmentationGreedyness() { assertMetrics("match:0.3717","a b c","a x b x x x x x x x x b c"); assertMetrics("match:0.4981","a b c","a x z x x x x x x x x b c"); @@ -715,7 +749,7 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { } else { float correctValue=Float.parseFloat(correctValueString); - assertEquals(metricName, correctValue, (float)Math.round(metrics.get(metricName)*10000)/10000 ); + assertEquals(metricName, correctValue, (float)Math.round(metrics.get(metricName)*10000)/10000, 0.000000001); } } } @@ -754,4 +788,5 @@ public class FieldMatchMetricsTestCase extends junit.framework.TestCase { } return new Field(terms.build()); } + } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/ReferenceTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/ReferenceTestCase.java index f275f95ca8e..4298beb5233 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/ReferenceTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/ReferenceTestCase.java @@ -5,9 +5,9 @@ import com.yahoo.searchlib.rankingexpression.rule.NameNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import org.junit.Test; -import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author bratseth diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/ContextReuseTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/ContextReuseTestCase.java index 3d5710b81e0..3ad5c0f7316 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/ContextReuseTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/ContextReuseTestCase.java @@ -6,19 +6,22 @@ import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.evaluation.ArrayContext; import com.yahoo.searchlib.rankingexpression.evaluation.ExpressionOptimizer; import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import org.junit.Test; import java.io.File; import java.io.IOException; +import static org.junit.Assert.assertEquals; + /** * This tests reuse of a optimized context which is not initialized with * all values referenced in the expression. * * @author bratseth */ -public class ContextReuseTestCase extends junit.framework.TestCase { +public class ContextReuseTestCase { - private String contextString= + private String contextString = "CONCEPTTYPE = 0.0\n" + "REGEXTYPE = 0.0\n" + "POS_18 = 0.0\n" + @@ -43,6 +46,7 @@ public class ContextReuseTestCase extends junit.framework.TestCase { "EXTENDEDTYPE = 0.0\n" + "ENTITYPLACETYPE = 0.0\n"; + @Test public void testIt() throws ParseException, IOException { // Prepare RankingExpression expression=new RankingExpression(IOUtils.readFile(new File("src/test/files/s-expression.vre"))); @@ -55,7 +59,7 @@ public class ContextReuseTestCase extends junit.framework.TestCase { String[] contextValueParts = contextValueString.split("="); context.put(contextValueParts[0].trim(), Double.valueOf(contextValueParts[1].trim())); } - assertEquals(-2.3450294999999994, expression.evaluate(context).asDouble()); + assertEquals(-2.3450294999999994, expression.evaluate(context).asDouble(), 0.000000000001); } } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestOptimizerTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestOptimizerTestCase.java index 420819a8b2a..ce78703f842 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestOptimizerTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestOptimizerTestCase.java @@ -4,12 +4,17 @@ package com.yahoo.searchlib.rankingexpression.evaluation.gbdtoptimization; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.evaluation.*; import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * @author bratseth */ -public class GBDTForestOptimizerTestCase extends junit.framework.TestCase { +public class GBDTForestOptimizerTestCase { + @Test public void testForestOptimization() throws ParseException { String gbdtString = "if (LW_NEWS_SEARCHES_RATIO < 1.72971, 0.0697159, if (LW_USERS < 0.10496, if (SEARCHES < 0.0329127, 0.151257, 0.117501), if (SUGG_OVERLAP < 18.5, 0.0897622, 0.0756903))) + \n" + @@ -55,6 +60,7 @@ public class GBDTForestOptimizerTestCase extends junit.framework.TestCase { assertEqualish(result3, oResult3); } + @Test public void testForestOptimizationWithSetMembershipConditions() throws ParseException { String gbdtString = "if (MYSTRING in [\"string 1\",\"string 2\"], 0.0697159, if (LW_USERS < 0.10496, if (SEARCHES < 0.0329127, 0.151257, 0.117501), if (MYSTRING in [\"string 2\"], 0.0897622, 0.0756903))) + \n" + diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTOptimizerTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTOptimizerTestCase.java index 1cbd19b667e..4b7462505fc 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTOptimizerTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTOptimizerTestCase.java @@ -7,96 +7,105 @@ import com.yahoo.searchlib.rankingexpression.evaluation.ExpressionOptimizer; import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; import com.yahoo.searchlib.rankingexpression.evaluation.OptimizationReport; import com.yahoo.searchlib.rankingexpression.parser.ParseException; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; /** * @author bratseth */ -public class GBDTOptimizerTestCase extends junit.framework.TestCase { +public class GBDTOptimizerTestCase { + + private final double delta = 0.00000000001; + @Test public void testSimpleNodeOptimization() throws ParseException { - RankingExpression gbdt=new RankingExpression("if (a < 2, if (b < 2, 5, 6), 4) + if (a < 3, 7, 8)"); + RankingExpression gbdt = new RankingExpression("if (a < 2, if (b < 2, 5, 6), 4) + if (a < 3, 7, 8)"); // Optimized evaluation - ArrayContext arguments=new ArrayContext(gbdt); - ExpressionOptimizer optimizer=new ExpressionOptimizer(); + ArrayContext arguments = new ArrayContext(gbdt); + ExpressionOptimizer optimizer = new ExpressionOptimizer(); optimizer.getOptimizer(GBDTForestOptimizer.class).setEnabled(false); - OptimizationReport report=optimizer.optimize(gbdt,arguments); - assertEquals(2,report.getMetric("Optimized GDBT trees")); - arguments.put("a",1d); - arguments.put("b",2d); - assertEquals(13.0,gbdt.evaluate(arguments).asDouble()); + OptimizationReport report = optimizer.optimize(gbdt,arguments); + assertEquals(2, report.getMetric("Optimized GDBT trees")); + arguments.put("a", 1d); + arguments.put("b", 2d); + assertEquals(13.0, gbdt.evaluate(arguments).asDouble(), delta); } + @Test public void testNodeOptimization() throws ParseException { - String gbdtString= + String gbdtString = "if (LW_NEWS_SEARCHES_RATIO < 1.72971, 0.0697159, if (LW_USERS < 0.10496, if (SEARCHES < 0.0329127, 0.151257, 0.117501), if (SUGG_OVERLAP < 18.5, 0.0897622, 0.0756903))) + \n" + "if (LW_NEWS_SEARCHES_RATIO < 1.73156, if (NEWS_USERS < 0.0737993, -0.00481646, 0.00110018), if (LW_USERS < 0.0844616, 0.0488919, if (SUGG_OVERLAP < 32.5, 0.0136917, 9.85328E-4))) + \n" + "if (LW_NEWS_SEARCHES_RATIO < 1.74451, -0.00298257, if (LW_USERS < 0.116207, if (SEARCHES < 0.0329127, 0.0676105, 0.0340198), if (NUM_WORDS < 1.5, -8.55514E-5, 0.0112406))) + \n" + "if (LW_NEWS_SEARCHES_RATIO < 1.72995, if (NEWS_USERS < 0.0737993, -0.00407515, 0.00139088), if (LW_USERS == 0.0509035, 0.0439466, if (LW_USERS < 0.325818, 0.0187156, 0.00236949)))"; - RankingExpression gbdt=new RankingExpression(gbdtString); + RankingExpression gbdt = new RankingExpression(gbdtString); // Regular evaluation - MapContext arguments=new MapContext(); - arguments.put("LW_NEWS_SEARCHES_RATIO",1d); - arguments.put("SUGG_OVERLAP",17d); - double result1=gbdt.evaluate(arguments).asDouble(); - arguments.put("LW_NEWS_SEARCHES_RATIO",2d); - arguments.put("SUGG_OVERLAP",20d); - double result2=gbdt.evaluate(arguments).asDouble(); - arguments.put("LW_NEWS_SEARCHES_RATIO",2d); - arguments.put("SUGG_OVERLAP",40d); - double result3=gbdt.evaluate(arguments).asDouble(); + MapContext arguments = new MapContext(); + arguments.put("LW_NEWS_SEARCHES_RATIO", 1d); + arguments.put("SUGG_OVERLAP", 17d); + double result1 = gbdt.evaluate(arguments).asDouble(); + arguments.put("LW_NEWS_SEARCHES_RATIO", 2d); + arguments.put("SUGG_OVERLAP", 20d); + double result2 = gbdt.evaluate(arguments).asDouble(); + arguments.put("LW_NEWS_SEARCHES_RATIO", 2d); + arguments.put("SUGG_OVERLAP", 40d); + double result3 = gbdt.evaluate(arguments).asDouble(); // Optimized evaluation - ArrayContext fArguments=new ArrayContext(gbdt); - ExpressionOptimizer optimizer=new ExpressionOptimizer(); + ArrayContext fArguments = new ArrayContext(gbdt); + ExpressionOptimizer optimizer = new ExpressionOptimizer(); optimizer.getOptimizer(GBDTForestOptimizer.class).setEnabled(false); - OptimizationReport report=optimizer.optimize(gbdt,fArguments); - assertEquals(4,report.getMetric("Optimized GDBT trees")); - fArguments.put("LW_NEWS_SEARCHES_RATIO",1d); - fArguments.put("SUGG_OVERLAP",17d); - double oResult1=gbdt.evaluate(fArguments).asDouble(); - fArguments.put("LW_NEWS_SEARCHES_RATIO",2d); - fArguments.put("SUGG_OVERLAP",20d); - double oResult2=gbdt.evaluate(fArguments).asDouble(); - fArguments.put("LW_NEWS_SEARCHES_RATIO",2d); - fArguments.put("SUGG_OVERLAP",40d); - double oResult3=gbdt.evaluate(fArguments).asDouble(); + OptimizationReport report = optimizer.optimize(gbdt,fArguments); + assertEquals(4, report.getMetric("Optimized GDBT trees")); + fArguments.put("LW_NEWS_SEARCHES_RATIO", 1d); + fArguments.put("SUGG_OVERLAP", 17d); + double oResult1 = gbdt.evaluate(fArguments).asDouble(); + fArguments.put("LW_NEWS_SEARCHES_RATIO", 2d); + fArguments.put("SUGG_OVERLAP", 20d); + double oResult2 = gbdt.evaluate(fArguments).asDouble(); + fArguments.put("LW_NEWS_SEARCHES_RATIO", 2d); + fArguments.put("SUGG_OVERLAP", 40d); + double oResult3 = gbdt.evaluate(fArguments).asDouble(); // Assert the same results are produced - assertEquals(result1,oResult1); - assertEquals(result2,oResult2); - assertEquals(result3,oResult3); + assertEquals(result1, oResult1, delta); + assertEquals(result2, oResult2, delta); + assertEquals(result3, oResult3, delta); } + @Test public void testFeatureNamesWithDots() throws ParseException { - String gbdtString= + String gbdtString = "if (a.b < 1.72971, 0.0697159, if (a.b.c < 0.10496, if (a.c < 0.0329127, 0.151257, 0.117501), if (a < 18.5, 0.0897622, 0.0756903))) + 1"; - RankingExpression gbdt=new RankingExpression(gbdtString); + RankingExpression gbdt = new RankingExpression(gbdtString); // Regular evaluation - MapContext arguments=new MapContext(); - arguments.put("a.b",1d); - arguments.put("a.b.c",0.1d); - arguments.put("a.c",0.01d); - arguments.put("a",19d); - double result=gbdt.evaluate(arguments).asDouble(); + MapContext arguments = new MapContext(); + arguments.put("a.b", 1d); + arguments.put("a.b.c", 0.1d); + arguments.put("a.c", 0.01d); + arguments.put("a", 19d); + double result = gbdt.evaluate(arguments).asDouble(); // Optimized evaluation - ArrayContext fArguments=new ArrayContext(gbdt); - OptimizationReport report=new OptimizationReport(); - new GBDTOptimizer().optimize(gbdt,fArguments,report); - assertEquals("Optimization result is as expected:\n" + report,1,report.getMetric("Optimized GDBT trees")); - fArguments.put("a.b",1d); - fArguments.put("a.b.c",0.1d); - fArguments.put("a.c",0.01d); - fArguments.put("a",19d); - double oResult=gbdt.evaluate(fArguments).asDouble(); + ArrayContext fArguments = new ArrayContext(gbdt); + OptimizationReport report = new OptimizationReport(); + new GBDTOptimizer().optimize(gbdt, fArguments, report); + assertEquals("Optimization result is as expected:\n" + report, 1, report.getMetric("Optimized GDBT trees")); + fArguments.put("a.b", 1d); + fArguments.put("a.b.c", 0.1d); + fArguments.put("a.c", 0.01d); + fArguments.put("a", 19d); + double oResult = gbdt.evaluate(fArguments).asDouble(); // Assert the same results are produced - assertEquals(result,oResult); + assertEquals(result, oResult, delta); } + @Test public void testBug4009433() throws ParseException { RankingExpression exp = new RankingExpression("10*if(two>35,if(two>one,if(two>=670,4,8),if(two>8000,5,3)),if(two==478,90,91))"); new GBDTOptimizer().optimize(exp, new ArrayContext(exp), new OptimizationReport()); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/DimensionRenamerTest.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/DimensionRenamerTest.java index ebcfde54c70..74b0d11f1d6 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/DimensionRenamerTest.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/integration/tensorflow/DimensionRenamerTest.java @@ -3,7 +3,7 @@ package com.yahoo.searchlib.rankingexpression.integration.tensorflow; import com.yahoo.searchlib.rankingexpression.integration.tensorflow.importer.DimensionRenamer; import org.junit.Test; -import static junit.framework.TestCase.assertTrue; +import static org.junit.Assert.assertTrue; public class DimensionRenamerTest { @@ -43,7 +43,6 @@ public class DimensionRenamerTest { assertTrue(secondDimensionOfXName.compareTo(firstDimensionOfWName) == 0); assertTrue(firstDimensionOfXName.compareTo(secondDimensionOfWName) < 0); assertTrue(secondDimensionOfWName.compareTo(firstDimensionOfBName) == 0); - - } + } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/treenet/TreeNetParserTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/treenet/TreeNetParserTestCase.java index cd7643b9f74..5f48109d4c6 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/treenet/TreeNetParserTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/treenet/TreeNetParserTestCase.java @@ -4,17 +4,24 @@ package com.yahoo.searchlib.treenet; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.treenet.parser.ParseException; import com.yahoo.searchlib.treenet.parser.TreeNetParser; -import junit.framework.TestCase; +import org.junit.Test; -import java.io.*; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.StringReader; + +import static org.junit.Assert.assertEquals; /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ -public class TreeNetParserTestCase extends TestCase { +public class TreeNetParserTestCase { private static final boolean WRITE_FILES = false; + @Test public void testRankingExpression() { for (int i = 1; i <= 8; ++i) { String inputFile = String.format("src/test/files/treenet%02d.model", i); @@ -76,4 +83,5 @@ public class TreeNetParserTestCase extends TestCase { throw new AssertionError(e); } } + } diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala index 50ad6c73daa..5271cd400d4 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/StandaloneContainerApplication.scala @@ -172,7 +172,6 @@ object StandaloneContainerApplication { deployState.getDocumentModel, deployState.getProperties.vespaVersion(), deployState.getProperties.applicationId()) - val spec = containerRootElement(applicationPackage) val containerModel = newContainerModelBuilder(networkingOption).build(deployState, configModelRepo, vespaRoot, spec) @@ -180,14 +179,10 @@ object StandaloneContainerApplication { DeprecationSuppressor.initializeContainerModel(containerModel, configModelRepo) val container = first(containerModel.getCluster().getContainers) - // TODO: If we can do the mutations below on the builder, we can separate out model finalization from the - // VespaModel constructor, such that the above and below code to finalize the container can be + // TODO: Separate out model finalization from the VespaModel constructor, + // such that the above and below code to finalize the container can be // replaced by root.finalize(); - // Always disable rpc server for standalone container. This server will soon be removed anyway. - container.setRpcServerEnabled(false) - container.setHttpServerEnabled(networkingOption == Networking.enable) - initializeContainer(container, spec) root.freezeModelTopology() diff --git a/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java b/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java index 91a19605286..b452c6eb204 100644 --- a/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/CounterGroupTestCase.java @@ -8,13 +8,18 @@ import java.util.logging.LogRecord; import java.util.logging.Logger; import com.yahoo.container.StatisticsConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; /** * Test set for groups of counters. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ -public class CounterGroupTestCase extends junit.framework.TestCase { +public class CounterGroupTestCase { + private volatile boolean gotRecord = false; private class CounterGroupHandler extends Handler { @@ -40,6 +45,7 @@ public class CounterGroupTestCase extends junit.framework.TestCase { } } + @Test public void testBasic() { Logger logger = Logger.getLogger(CounterGroup.class.getName()); boolean initUseParentHandlers = logger.getUseParentHandlers(); @@ -64,6 +70,7 @@ public class CounterGroupTestCase extends junit.framework.TestCase { logger.setUseParentHandlers(initUseParentHandlers); } + @Test public void testObjectContracts() { CounterGroup c = new CounterGroup("test", Statistics.nullImplementation, false); CounterGroup c2 = new CounterGroup("test", Statistics.nullImplementation, false); @@ -77,6 +84,7 @@ public class CounterGroupTestCase extends junit.framework.TestCase { c.equals(c2)); } + @Test public void testConfigStuff() { Logger logger = Logger.getLogger(CounterGroup.class.getName()); boolean initUseParentHandlers = logger.getUseParentHandlers(); diff --git a/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java b/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java index 56258f60fc7..b33586a6428 100644 --- a/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/CounterTestCase.java @@ -1,20 +1,24 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.statistics; - import java.util.Arrays; import java.util.logging.Logger; import com.yahoo.container.StatisticsConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; /** * Check counters work. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ -public class CounterTestCase extends junit.framework.TestCase { +public class CounterTestCase { + @Test public void testBasic() { Counter c = new Counter("test", Statistics.nullImplementation, false); c.increment(); @@ -32,6 +36,7 @@ public class CounterTestCase extends junit.framework.TestCase { assertEquals(0, c.get()); } + @Test public void testObjectContracts() { final String counterName = "test"; Counter c = new Counter(counterName, Statistics.nullImplementation, false); @@ -51,6 +56,7 @@ public class CounterTestCase extends junit.framework.TestCase { assertEquals(prefix, image.substring(0, prefix.length())); } + @Test public void testConfigStuff() { Logger logger = Logger.getLogger(Counter.class.getName()); boolean initUseParentHandlers = logger.getUseParentHandlers(); diff --git a/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java b/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java index 3d1edf8616b..6b970220b74 100644 --- a/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/HistogramTestCase.java @@ -2,20 +2,20 @@ package com.yahoo.statistics; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * Some low level checking of the histograms. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ -public class HistogramTestCase extends junit.framework.TestCase { - - public HistogramTestCase(String name) { - super(name); - } +public class HistogramTestCase { + @Test public void testFindBucket() { Limits l = new Limits(); double[] thresholds = {.5, 1.0, 5.0}; @@ -32,6 +32,7 @@ public class HistogramTestCase extends junit.framework.TestCase { } + @Test public void testMerge() { Limits l = new Limits(); double[] thresholds = {.5, 1.0, 5.0}; @@ -46,6 +47,7 @@ public class HistogramTestCase extends junit.framework.TestCase { assertEquals("(0) < 0.5 (3) < 1.0 (0) < 5.0 (0)", h2.toString()); } + @Test public void testMultiDimensionalMerge() { Limits l = new Limits(); double[] thresholds = {.5, 1.0, 5.0}; @@ -63,6 +65,7 @@ public class HistogramTestCase extends junit.framework.TestCase { h.toString()); } + @Test public void testEmptyHistogram() { try { new Histogram(new Limits()); diff --git a/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java b/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java index eabfb61acd3..b92182c3534 100644 --- a/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/ValueGroupTestCase.java @@ -7,13 +7,19 @@ import java.util.logging.LogRecord; import java.util.logging.Logger; import com.yahoo.container.StatisticsConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Test set for groups of values. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ -public class ValueGroupTestCase extends junit.framework.TestCase { +public class ValueGroupTestCase { + private volatile boolean gotRecord = false; private class ValueGroupHandler extends Handler { @@ -39,6 +45,7 @@ public class ValueGroupTestCase extends junit.framework.TestCase { } } + @Test public void testBasic() { Logger logger = Logger.getLogger(ValueGroup.class.getName()); boolean initUseParentHandlers = logger.getUseParentHandlers(); @@ -64,6 +71,7 @@ public class ValueGroupTestCase extends junit.framework.TestCase { logger.setUseParentHandlers(initUseParentHandlers); } + @Test public void testOverlappingSubnames() { final MockStatistics manager = new MockStatistics(); ValueGroup v = new ValueGroup("jappe", manager); @@ -76,6 +84,7 @@ public class ValueGroupTestCase extends junit.framework.TestCase { assertEquals(2, manager.registerCount); } + @Test public void testObjectContracts() { ValueGroup v = new ValueGroup("test", new MockStatistics()); ValueGroup v2 = new ValueGroup("test", new MockStatistics()); diff --git a/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java b/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java index 2804442a982..5650d8125b6 100644 --- a/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java +++ b/statistics/src/test/java/com/yahoo/statistics/ValueTestCase.java @@ -9,14 +9,22 @@ import java.util.logging.Logger; import com.yahoo.container.StatisticsConfig; import static com.yahoo.container.StatisticsConfig.Values.Operations; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + import com.yahoo.statistics.Value.Parameters; +import org.junit.Test; /** * Check correct statistics are generated for basic values. * - * @author <a href="mailto:steinar@yahoo-inc.com">Steinar Knutsen</a> + * @author Steinar Knutsen */ -public class ValueTestCase extends junit.framework.TestCase { +public class ValueTestCase { + + private static final double delta = 0.0000000001; private static final String NALLE = "nalle"; private static final double SECOND = 43.0d; @@ -36,6 +44,7 @@ public class ValueTestCase extends junit.framework.TestCase { } + @Test public void testMean() { Value v = new Value("thingie", Statistics.nullImplementation, new Parameters().setLogMean(true)); v.put(1.0); @@ -48,6 +57,7 @@ public class ValueTestCase extends junit.framework.TestCase { assertTrue("Value should have been reset.", 0.0d == v.getMean()); } + @Test public void testMin() { Value v = new Value("thingie", Statistics.nullImplementation, new Parameters().setLogMin(true)); v.put(2.0); @@ -59,6 +69,7 @@ public class ValueTestCase extends junit.framework.TestCase { assertTrue("Min should be -1.0", -1.0 == v.getMin()); } + @Test public void testMax() { Value v = new Value("thingie", Statistics.nullImplementation, new Parameters().setLogMax(true)); v.put(-1.0); @@ -71,6 +82,7 @@ public class ValueTestCase extends junit.framework.TestCase { assertTrue("Max should be 4.0", 4.0 == v.getMax()); } + @Test public void testHistogram() { Value v = new Value("thingie", Statistics.nullImplementation, new Parameters() .setLogHistogram(true).setHistogramId(HistogramType.REGULAR) @@ -84,6 +96,7 @@ public class ValueTestCase extends junit.framework.TestCase { " thingie (1) < 0.0 (1) < 1.0 (1) < 2.0 (2)")); } + @Test public void testCallback() { Logger logger = Logger.getLogger(Value.class.getName()); boolean initUseParentHandlers = logger.getUseParentHandlers(); @@ -91,14 +104,15 @@ public class ValueTestCase extends junit.framework.TestCase { Value v = new Value("thingie", Statistics.nullImplementation, new Parameters() .setLogRaw(true).setCallback(new TrivialCallback())); v.run(); - assertEquals(FIRST, v.get()); + assertEquals(FIRST, v.get(), delta); v.run(); - assertEquals(SECOND, v.get()); + assertEquals(SECOND, v.get(), delta); v.run(); - assertEquals(SECOND, v.get()); + assertEquals(SECOND, v.get(), delta); logger.setUseParentHandlers(initUseParentHandlers); } + @Test public void testParameter() { Value.Parameters p = new Value.Parameters().setLogInsertions(true) .setNameExtension(true).setAppendChar('_'); @@ -148,6 +162,7 @@ public class ValueTestCase extends junit.framework.TestCase { } } + @Test public void testParameterFromConfig() { Logger logger = Logger.getLogger(Value.class.getName()); boolean initUseParentHandlers = logger.getUseParentHandlers(); @@ -173,13 +188,14 @@ public class ValueTestCase extends junit.framework.TestCase { Value v = Value.buildValue(NALLE, m, null); final double x = 79.0d; v.put(x); - assertEquals(x, v.getMean()); + assertEquals(x, v.getMean(), delta); v.run(); assertEquals(true, h.gotRecord); logger.removeHandler(h); logger.setUseParentHandlers(initUseParentHandlers); } + @Test public void testReverseHistogram() { Logger logger = Logger.getLogger(Value.class.getName()); boolean initUseParentHandlers = logger.getUseParentHandlers(); @@ -209,6 +225,7 @@ public class ValueTestCase extends junit.framework.TestCase { logger.setUseParentHandlers(initUseParentHandlers); } + @Test public void testCumulativeHistogram() { Logger logger = Logger.getLogger(Value.class.getName()); boolean initUseParentHandlers = logger.getUseParentHandlers(); @@ -238,6 +255,7 @@ public class ValueTestCase extends junit.framework.TestCase { logger.setUseParentHandlers(initUseParentHandlers); } + @Test public void testObjectContracts() { final String valueName = "test"; Value v = new Value(valueName, Statistics.nullImplementation, Value.defaultParameters()); @@ -256,7 +274,6 @@ public class ValueTestCase extends junit.framework.TestCase { assertEquals(valueName, image.substring(image.length() - valueName.length())); } - public class MockStatistics implements Statistics { public StatisticsConfig config = null; public int registerCount = 0; diff --git a/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java index 0982a49bee1..5e1d811ade1 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/BucketDistributionTestCase.java @@ -2,18 +2,22 @@ package com.yahoo.vdslib; import com.yahoo.document.BucketId; +import org.junit.Test; import java.util.Random; +import static org.junit.Assert.assertEquals; + /** - * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen</a> + * @author Simon Thoresen */ -public class BucketDistributionTestCase extends junit.framework.TestCase { +public class BucketDistributionTestCase { private static final int NUM_COLUMNS = 16; private static final int MIN_BUCKETBITS = 4; private static final int MAX_BUCKETBITS = 9; + @Test public void printExpected() { System.out.println(" int[][] expected = new int[][] {"); for (int numBucketBits = MIN_BUCKETBITS; numBucketBits <= MAX_BUCKETBITS; ++numBucketBits) { @@ -38,6 +42,7 @@ public class BucketDistributionTestCase extends junit.framework.TestCase { System.out.println(" };"); } + @Test public void testDistribution() { int[][] expected = new int[][] { new int[] { @@ -90,6 +95,7 @@ public class BucketDistributionTestCase extends junit.framework.TestCase { } } + @Test public void testNumBucketBits() { Random rnd = new Random(); @@ -108,4 +114,5 @@ public class BucketDistributionTestCase extends junit.framework.TestCase { assertEquals(0, bd.getColumn(new BucketId(32, (rnd.nextLong() << 16) & i))); } } + } diff --git a/vdslib/src/test/java/com/yahoo/vdslib/DocumentListTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/DocumentListTestCase.java index 62429f7c300..eaf9e576a5b 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/DocumentListTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/DocumentListTestCase.java @@ -3,21 +3,23 @@ package com.yahoo.vdslib; import com.yahoo.document.*; import com.yahoo.document.datatypes.FloatFieldValue; -import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.serialization.DocumentSerializer; import com.yahoo.document.serialization.DocumentSerializerFactory; import com.yahoo.document.update.FieldUpdate; +import org.junit.Test; -import java.io.EOFException; -import java.io.FileInputStream; import java.io.FileOutputStream; -import java.io.IOException; import java.util.ArrayList; import java.util.List; -public class DocumentListTestCase extends junit.framework.TestCase { +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +public class DocumentListTestCase { + + @Test @SuppressWarnings("deprecation") public void testSelfSerializationAndWriteJavaFile() throws Exception { DocumentTypeManager docMan = new DocumentTypeManager(); @@ -34,7 +36,7 @@ public class DocumentListTestCase extends junit.framework.TestCase { DocumentUpdate docUp = new DocumentUpdate(docMan.getDocumentType("benchmark"), new DocumentId("userdoc:foo:99999999:4")); docUp.addFieldUpdate(FieldUpdate.createAssign(docUp.getType().getField("bodystring"), new StringFieldValue("ballooooo"))); - List<Entry> entries = new ArrayList<Entry>(); + List<Entry> entries = new ArrayList<>(); entries.add(Entry.create(put1)); entries.add(Entry.create(doc2)); entries.add(Entry.create(put3)); @@ -92,6 +94,7 @@ public class DocumentListTestCase extends junit.framework.TestCase { assertEquals(new StringFieldValue("ballooooo"),((DocumentUpdate) entry4.getDocumentOperation()).getFieldUpdate(0).getValueUpdate(0).getValue()); } + @Test public void testContains() { DocumentTypeManager manager = new DocumentTypeManager(); DocumentTypeManagerConfigurer.configure(manager, "file:src/test/files/documentmanager.cfg"); @@ -119,17 +122,17 @@ public class DocumentListTestCase extends junit.framework.TestCase { DocumentList documentList2 = DocumentList.create(entries2); - assert(documentList.containsAll(documentList2)); + assertTrue(documentList.containsAll(documentList2)); Long t = put4.getDocument().getLastModified(); put4.getDocument().setLastModified(13L); - assert(!documentList.containsAll(documentList2)); + assertTrue(!documentList.containsAll(documentList2)); put4.getDocument().setLastModified(t); assert(documentList.containsAll(documentList2)); entries.remove(2); - assert(!documentList.containsAll(documentList2)); - + assertTrue(!documentList.containsAll(documentList2)); } + } diff --git a/vdslib/src/test/java/com/yahoo/vdslib/EntryTestCase.java b/vdslib/src/test/java/com/yahoo/vdslib/EntryTestCase.java index 0f650185d69..ccce2ffcbb7 100644 --- a/vdslib/src/test/java/com/yahoo/vdslib/EntryTestCase.java +++ b/vdslib/src/test/java/com/yahoo/vdslib/EntryTestCase.java @@ -1,17 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vdslib; -import com.yahoo.document.Document; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; +import org.junit.Test; + +import static org.junit.Assert.assertTrue; /** * @author banino */ -public class EntryTestCase extends junit.framework.TestCase{ +public class EntryTestCase { + @Test public void testEquals() { DocumentTypeManager manager = new DocumentTypeManager(); DocumentTypeManagerConfigurer.configure(manager, "file:src/test/files/documentmanager.cfg"); @@ -22,13 +25,13 @@ public class EntryTestCase extends junit.framework.TestCase{ Entry entry1 = Entry.create(put1); Entry entry2 = Entry.create(put1); - assert(entry1.equals(entry2)); + assertTrue(entry1.equals(entry2)); Entry entry3 = Entry.create(put2); - assert(!entry1.equals(entry3)); - + assertTrue(!entry1.equals(entry3)); } + @Test public void testHashCode() { DocumentTypeManager manager = new DocumentTypeManager(); DocumentTypeManagerConfigurer.configure(manager, "file:src/test/files/documentmanager.cfg"); @@ -39,13 +42,10 @@ public class EntryTestCase extends junit.framework.TestCase{ Entry entry1 = Entry.create(put1); Entry entry2 = Entry.create(put1); - assert(entry1.hashCode() == entry2.hashCode()); + assertTrue(entry1.hashCode() == entry2.hashCode()); Entry entry3 = Entry.create(put2); - assert(entry1.hashCode() != entry3.hashCode()); - + assertTrue(entry1.hashCode() != entry3.hashCode()); } - - } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java index 26fe0b6e930..4278e641166 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzCredentialsService.java @@ -35,16 +35,16 @@ class AthenzCredentialsService { private final IdentityConfig identityConfig; private final IdentityDocumentService identityDocumentService; - private final AthenzService athenzService; + private final ZtsClient ztsClient; private final File trustStoreJks; AthenzCredentialsService(IdentityConfig identityConfig, IdentityDocumentService identityDocumentService, - AthenzService athenzService, + ZtsClient ztsClient, File trustStoreJks) { this.identityConfig = identityConfig; this.identityDocumentService = identityDocumentService; - this.athenzService = athenzService; + this.ztsClient = ztsClient; this.trustStoreJks = trustStoreJks; } @@ -64,13 +64,12 @@ class AthenzCredentialsService { identityConfig.service(), rawDocument, Pkcs10CsrUtils.toPem(csr)); - InstanceIdentity instanceIdentity = athenzService.sendInstanceRegisterRequest(instanceRegisterInformation, - document.ztsEndpoint); + InstanceIdentity instanceIdentity = ztsClient.sendInstanceRegisterRequest(instanceRegisterInformation, + document.ztsEndpoint); return toAthenzCredentials(instanceIdentity, keyPair, document); } - AthenzCredentials updateCredentials(AthenzCredentials currentCredentials) { - SignedIdentityDocument document = currentCredentials.getIdentityDocument(); + AthenzCredentials updateCredentials(SignedIdentityDocument document, SSLContext sslContext) { KeyPair newKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); Pkcs10Csr csr = createCSR(identityConfig.domain(), identityConfig.service(), @@ -80,14 +79,13 @@ class AthenzCredentialsService { newKeyPair); InstanceRefreshInformation refreshInfo = new InstanceRefreshInformation(Pkcs10CsrUtils.toPem(csr)); InstanceIdentity instanceIdentity = - athenzService.sendInstanceRefreshRequest(document.providerService, - identityConfig.domain(), - identityConfig.service(), - document.providerUniqueId, - refreshInfo, - document.ztsEndpoint, - currentCredentials.getCertificate(), - currentCredentials.getKeyPair().getPrivate()); + ztsClient.sendInstanceRefreshRequest(document.providerService, + identityConfig.domain(), + identityConfig.service(), + document.providerUniqueId, + refreshInfo, + document.ztsEndpoint, + sslContext); return toAthenzCredentials(instanceIdentity, newKeyPair, document); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index 594fa91e18f..eb0ae89fdcf 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -8,11 +8,17 @@ import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; import com.yahoo.container.jdisc.athenz.AthenzIdentityProviderException; import com.yahoo.jdisc.Metric; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; +import com.yahoo.vespa.athenz.tls.KeyStoreType; +import com.yahoo.vespa.athenz.tls.SslContextBuilder; import com.yahoo.vespa.defaults.Defaults; import javax.net.ssl.SSLContext; import java.io.File; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -37,20 +43,22 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen public static final String CERTIFICATE_EXPIRY_METRIC_NAME = "athenz-tenant-cert.expiry.seconds"; private volatile AthenzCredentials credentials; + private final ZtsClient ztsClient = new ZtsClient(); private final Metric metric; private final AthenzCredentialsService athenzCredentialsService; private final ScheduledExecutorService scheduler; private final Clock clock; - private final com.yahoo.vespa.athenz.api.AthenzService identity; + private final AthenzService identity; + // TODO IdentityConfig should contain ZTS uri and dns suffix @Inject public AthenzIdentityProviderImpl(IdentityConfig config, Metric metric) { this(config, metric, new AthenzCredentialsService(config, new IdentityDocumentService(config.loadBalancerAddress()), - new AthenzService(), - new File(Defaults.getDefaults().underVespaHome("share/ssl/certs/yahoo_certificate_bundle.jks"))), + new ZtsClient(), + getDefaultTrustStoreLocation()), new ScheduledThreadPoolExecutor(1), Clock.systemUTC()); } @@ -65,7 +73,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen this.athenzCredentialsService = athenzCredentialsService; this.scheduler = scheduler; this.clock = clock; - this.identity = new com.yahoo.vespa.athenz.api.AthenzService(config.domain(), config.service()); + this.identity = new AthenzService(config.domain(), config.service()); registerInstance(); } @@ -80,7 +88,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } @Override - public com.yahoo.vespa.athenz.api.AthenzService identity() { + public AthenzService identity() { return identity; } @@ -100,6 +108,45 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } @Override + public SSLContext getRoleSslContext(String domain, String role) { + // This ssl context should ideally be cached as it is quite expensive to create. + PrivateKey privateKey = credentials.getKeyPair().getPrivate(); + X509Certificate roleCertificate = ztsClient.getRoleCertificate( + new AthenzDomain(domain), + role, + credentials.getIdentityDocument().dnsSuffix, + credentials.getIdentityDocument().ztsEndpoint, + identity, + privateKey, + credentials.getIdentitySslContext()); + return new SslContextBuilder() + .withKeyStore(privateKey, roleCertificate) + .withTrustStore(getDefaultTrustStoreLocation(), KeyStoreType.JKS) + .build(); + } + + @Override + public String getRoleToken(String domain) { + return ztsClient + .getRoleToken( + new AthenzDomain(domain), + credentials.getIdentityDocument().ztsEndpoint, + credentials.getIdentitySslContext()) + .getRawToken(); + } + + @Override + public String getRoleToken(String domain, String role) { + return ztsClient + .getRoleToken( + new AthenzDomain(domain), + role, + credentials.getIdentityDocument().ztsEndpoint, + credentials.getIdentitySslContext()) + .getRawToken(); + } + + @Override public void deconstruct() { try { scheduler.shutdownNow(); @@ -109,6 +156,10 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen } } + private static File getDefaultTrustStoreLocation() { + return new File(Defaults.getDefaults().underVespaHome("share/ssl/certs/yahoo_certificate_bundle.jks")); + } + private boolean isExpired(AthenzCredentials credentials) { return clock.instant().isAfter(getExpirationTime(credentials)); } @@ -121,7 +172,7 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen try { AthenzCredentials newCredentials = isExpired(credentials) ? athenzCredentialsService.registerInstance() - : athenzCredentialsService.updateCredentials(credentials); + : athenzCredentialsService.updateCredentials(credentials.getIdentityDocument(), credentials.getIdentitySslContext()); credentials = newCredentials; } catch (Throwable t) { log.log(LogLevel.WARNING, "Failed to update credentials: " + t.getMessage(), t); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentService.java index 34b28e48914..0d82bb29edd 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/IdentityDocumentService.java @@ -74,8 +74,7 @@ public class IdentityDocumentService { .setScheme("https") .setHost(loadBalancerName) .setPort(4443) - .setPath("/athenz/v1/provider/identity-document") - .addParameter("hostname", Defaults.getDefaults().vespaHostname()) + .setPath("/athenz/v1/provider/identity-document/tenant/" + Defaults.getDefaults().vespaHostname()) .build(); } catch (URISyntaxException e) { throw new RuntimeException(e); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzService.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ZtsClient.java index 98307a8a2d1..c995bfba791 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzService.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/ZtsClient.java @@ -1,13 +1,19 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.identityprovider.client; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.yahoo.athenz.zts.RoleCertificateRequest; +import com.yahoo.athenz.zts.RoleToken; +import com.yahoo.athenz.zts.ZTSClient; +import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.api.ZToken; +import com.yahoo.vespa.athenz.tls.X509CertificateUtils; import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; -import org.apache.http.conn.ssl.SSLContextBuilder; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; @@ -20,21 +26,15 @@ import javax.net.ssl.SSLContext; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; -import java.security.UnrecoverableKeyException; -import java.security.cert.Certificate; -import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.time.Duration; /** * @author mortent * @author bjorncs */ -public class AthenzService { +class ZtsClient { private static final String INSTANCE_API_PATH = "/zts/v1/instance"; @@ -44,7 +44,7 @@ public class AthenzService { /** * Send instance register request to ZTS, get InstanceIdentity */ - public InstanceIdentity sendInstanceRegisterRequest(InstanceRegisterInformation instanceRegisterInformation, + InstanceIdentity sendInstanceRegisterRequest(InstanceRegisterInformation instanceRegisterInformation, URI uri) { try(CloseableHttpClient client = HttpClientBuilder.create().setRetryHandler(retryHandler).build()) { HttpUriRequest postRequest = RequestBuilder.post() @@ -57,15 +57,14 @@ public class AthenzService { } } - public InstanceIdentity sendInstanceRefreshRequest(String providerService, - String instanceDomain, - String instanceServiceName, - String instanceId, - InstanceRefreshInformation instanceRefreshInformation, - URI ztsEndpoint, - X509Certificate certicate, - PrivateKey privateKey) { - try (CloseableHttpClient client = createHttpClientWithTlsAuth(certicate, privateKey, retryHandler)) { + InstanceIdentity sendInstanceRefreshRequest(String providerService, + String instanceDomain, + String instanceServiceName, + String instanceId, + InstanceRefreshInformation instanceRefreshInformation, + URI ztsEndpoint, + SSLContext sslContext) { + try (CloseableHttpClient client = createHttpClientWithTlsAuth(sslContext, retryHandler)) { URI uri = ztsEndpoint .resolve(INSTANCE_API_PATH + '/') .resolve(providerService + '/') @@ -82,6 +81,43 @@ public class AthenzService { } } + ZToken getRoleToken(AthenzDomain domain, + URI ztsEndpoint, + SSLContext sslContext) { + // TODO ztsEndpoint should contain '/zts/v1' as path + URI correctedZtsEndpoint = ztsEndpoint.resolve("/zts/v1"); + return new ZToken( + new ZTSClient(correctedZtsEndpoint.toString(), sslContext) + .getRoleToken(domain.getName()).getToken()); + } + + ZToken getRoleToken(AthenzDomain domain, + String roleName, + URI ztsEndpoint, + SSLContext sslContext) { + // TODO ztsEndpoint should contain '/zts/v1' as path + URI correctedZtsEndpoint = ztsEndpoint.resolve("/zts/v1"); + return new ZToken( + new ZTSClient(correctedZtsEndpoint.toString(), sslContext) + .getRoleToken(domain.getName(), roleName).getToken()); + } + + X509Certificate getRoleCertificate(AthenzDomain roleDomain, + String roleName, + String dnsSuffix, + URI ztsEndpoint, + AthenzService identity, + PrivateKey privateKey, + SSLContext sslContext) { + // TODO ztsEndpoint should contain '/zts/v1' as path + URI correctedZtsEndpoint = ztsEndpoint.resolve("/zts/v1"); + ZTSClient ztsClient = new ZTSClient(correctedZtsEndpoint.toString(), sslContext); + RoleCertificateRequest rcr = ZTSClient.generateRoleCertificateRequest( + identity.getDomain().getName(), identity.getName(), roleDomain.getName(), roleName, privateKey, dnsSuffix, (int) Duration.ofHours(1).getSeconds()); + RoleToken pemCert = ztsClient.postRoleCertificateRequest(roleDomain.getName(), roleName, rcr); + return X509CertificateUtils.fromPem(pemCert.token); + } + private InstanceIdentity getInstanceIdentity(CloseableHttpClient client, HttpUriRequest postRequest) throws IOException { try (CloseableHttpResponse response = client.execute(postRequest)) { @@ -99,26 +135,11 @@ public class AthenzService { return new StringEntity(objectMapper.writeValueAsString(value), ContentType.APPLICATION_JSON); } - private static CloseableHttpClient createHttpClientWithTlsAuth(X509Certificate certificate, - PrivateKey privateKey, + private static CloseableHttpClient createHttpClientWithTlsAuth(SSLContext sslContext, HttpRequestRetryHandler retryHandler) { - try { - String dummyPassword = "athenz"; - KeyStore keyStore = KeyStore.getInstance("JKS"); - keyStore.load(null); - keyStore.setKeyEntry("athenz", privateKey, dummyPassword.toCharArray(), new Certificate[]{certificate}); - SSLContext sslContext = new SSLContextBuilder() - .loadKeyMaterial(keyStore, dummyPassword.toCharArray()) - .build(); return HttpClientBuilder.create() .setRetryHandler(retryHandler) .setSslcontext(sslContext) .build(); - } catch (KeyStoreException | UnrecoverableKeyException | NoSuchAlgorithmException | - KeyManagementException | CertificateException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new UncheckedIOException(e); - } } } diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java index a0ae6ca61db..91c68702e87 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImplTest.java @@ -58,26 +58,25 @@ public class AthenzIdentityProviderImplTest { @Test public void metrics_updated_on_refresh() throws IOException { IdentityDocumentService identityDocumentService = mock(IdentityDocumentService.class); - AthenzService athenzService = mock(AthenzService.class); + ZtsClient ztsClient = mock(ZtsClient.class); ManualClock clock = new ManualClock(Instant.EPOCH); Metric metric = mock(Metric.class); when(identityDocumentService.getSignedIdentityDocument()).thenReturn(getIdentityDocument()); - when(athenzService.sendInstanceRegisterRequest(any(), any())).then(new Answer<InstanceIdentity>() { + when(ztsClient.sendInstanceRegisterRequest(any(), any())).then(new Answer<InstanceIdentity>() { @Override public InstanceIdentity answer(InvocationOnMock invocationOnMock) throws Throwable { return new InstanceIdentity(getCertificate(getExpirationSupplier(clock)), "TOKEN"); } }); - when(athenzService.sendInstanceRefreshRequest(anyString(), anyString(), anyString(), - anyString(), any(), any(), any(), any())) + when(ztsClient.sendInstanceRefreshRequest(anyString(), anyString(), anyString(), anyString(), any(), any(), any())) .thenThrow(new RuntimeException("#1")) .thenThrow(new RuntimeException("#2")) .thenReturn(new InstanceIdentity(getCertificate(getExpirationSupplier(clock)), "TOKEN")); AthenzCredentialsService credentialService = - new AthenzCredentialsService(IDENTITY_CONFIG, identityDocumentService, athenzService, createDummyTrustStore()); + new AthenzCredentialsService(IDENTITY_CONFIG, identityDocumentService, ztsClient, createDummyTrustStore()); AthenzIdentityProviderImpl identityProvider = new AthenzIdentityProviderImpl(IDENTITY_CONFIG, metric, credentialService, mock(ScheduledExecutorService.class), clock); |