diff options
244 files changed, 5268 insertions, 3116 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index d0ec98e3297..718e6172910 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -56,7 +56,7 @@ public interface ModelContext { boolean useFdispatchByDefault(); boolean dispatchWithProtobuf(); boolean useAdaptiveDispatch(); - // TODO: Remove when 7.33 is the oldest model in use + // TODO: Remove when 7.40 is the oldest model in use default boolean useSeparateServiceTypeForLogserverContainer() { return true; } boolean enableMetricsProxyContainer(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index 09bc29b181a..e94fa9bf040 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -3,10 +3,8 @@ package com.yahoo.vespa.model.admin; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.vespa.model.container.Container; -import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.vespa.model.container.component.AccessLogComponent; -import com.yahoo.vespa.model.container.component.Handler; /** * Container that should be running on same host as the logserver. Sets up a handler for getting logs from logserver. @@ -14,17 +12,14 @@ import com.yahoo.vespa.model.container.component.Handler; */ public class LogserverContainer extends Container { - private final boolean useSeparateServiceTypeForLogserverContainer; - - public LogserverContainer(AbstractConfigProducer parent, boolean useSeparateServiceTypeForLogserverContainer) { + public LogserverContainer(AbstractConfigProducer parent) { super(parent, "" + 0, 0); - this.useSeparateServiceTypeForLogserverContainer = useSeparateServiceTypeForLogserverContainer; addComponent(new AccessLogComponent(AccessLogComponent.AccessLogType.jsonAccessLog, ((LogserverContainerCluster) parent).getName(), true)); } @Override public ContainerServiceType myServiceType() { - return useSeparateServiceTypeForLogserverContainer ? ContainerServiceType.LOGSERVER_CONTAINER : ContainerServiceType.CONTAINER; + return ContainerServiceType.LOGSERVER_CONTAINER; } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 7d19c1d77c9..8980c7db93d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -110,7 +110,7 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { ContainerModel logserverClusterModel = new ContainerModel(context.withParent(admin).withId(logServerCluster.getSubId())); logserverClusterModel.setCluster(logServerCluster); - LogserverContainer container = new LogserverContainer(logServerCluster, deployState.getProperties().useSeparateServiceTypeForLogserverContainer()); + LogserverContainer container = new LogserverContainer(logServerCluster); container.setHostResource(hostResource); container.initService(deployState.getDeployLogger()); logServerCluster.addContainer(container); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index 2de11be08f2..db01bb91b3b 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -24,7 +24,7 @@ public enum SystemName { Public, /** VaaS */ - vaas; + vaas; // TODO: Remove this and use public everywhere public static SystemName defaultSystem() { return main; @@ -35,7 +35,7 @@ public enum SystemName { case "dev": return dev; case "cd": return cd; case "main": return main; - case "public": return Public; + case "public": case "Public": return Public; case "vaas": return vaas; default: throw new IllegalArgumentException(String.format("'%s' is not a valid system", value)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 53d20379989..bea087f6ae9 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -54,6 +54,7 @@ import com.yahoo.vespa.config.server.session.SilentDeployLogger; import com.yahoo.vespa.config.server.tenant.Rotations; import com.yahoo.vespa.config.server.tenant.Tenant; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.orchestrator.Orchestrator; import java.io.File; @@ -525,6 +526,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public long createSession(ApplicationId applicationId, TimeoutBudget timeoutBudget, File applicationDirectory) { Tenant tenant = tenantRepository.getTenant(applicationId.tenant()); + tenant.getApplicationRepo().createApplication(applicationId); LocalSessionRepo localSessionRepo = tenant.getLocalSessionRepo(); SessionFactory sessionFactory = tenant.getSessionFactory(); LocalSession session = sessionFactory.createSession(applicationDirectory, applicationId, timeoutBudget); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 656030bb8d2..348e8f1cafc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -32,6 +32,7 @@ import java.util.stream.Collectors; * of whatever session may be activated next, if any, and /lock is used for synchronizing writes to all these paths. * * @author Ulf Lilleengen + * @author jonmv */ public class TenantApplications { @@ -58,11 +59,7 @@ public class TenantApplications { } public static TenantApplications create(Curator curator, ReloadHandler reloadHandler, TenantName tenant) { - try { - return new TenantApplications(curator, TenantRepository.getApplicationsPath(tenant), reloadHandler, tenant); - } catch (Exception e) { - throw new RuntimeException(TenantRepository.logPre(tenant) + "Error creating application repo", e); - } + return new TenantApplications(curator, TenantRepository.getApplicationsPath(tenant), reloadHandler, tenant); } /** @@ -73,6 +70,7 @@ public class TenantApplications { public List<ApplicationId> activeApplications() { return curator.getChildren(applicationsPath).stream() .filter(this::isValid) + .sorted() .map(ApplicationId::fromSerializedForm) .filter(id -> activeSessionOf(id).isPresent()) .collect(Collectors.toUnmodifiableList()); @@ -109,11 +107,14 @@ public class TenantApplications { * @param sessionId Id of the session containing the application package for this id. */ public Transaction createPutTransaction(ApplicationId applicationId, long sessionId) { - if (curator.exists(applicationPath(applicationId))) { - return new CuratorTransaction(curator).add(CuratorOperations.setData(applicationPath(applicationId).getAbsolute(), Utf8.toAsciiBytes(sessionId))); - } else { - return new CuratorTransaction(curator).add(CuratorOperations.create(applicationPath(applicationId).getAbsolute(), Utf8.toAsciiBytes(sessionId))); - } + return new CuratorTransaction(curator).add(CuratorOperations.setData(applicationPath(applicationId).getAbsolute(), Utf8.toAsciiBytes(sessionId))); + } + + /** + * Creates a node for the given application, marking its existence. + */ + public void createApplication(ApplicationId id) { + curator.create(applicationPath(id)); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 082be2583c2..01bb4e2dc76 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -115,7 +115,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { /** Activates this. If it is not already prepared, this will call prepare first. */ @Override public void activate() { - if (! prepared) + if ( ! prepared) prepare(); TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java index 77572856ff5..a7f8b8164a5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java @@ -36,6 +36,7 @@ public class HostRegistry<T> implements HostValidator<T> { addHosts(key, newHosts); } + @Override public synchronized void verifyHosts(T key, Collection<String> newHosts) { for (String host : newHosts) { if (hostAlreadyTaken(host, key)) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java index af8956803ab..6a0a4a19737 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/LocalSession.java @@ -64,6 +64,7 @@ public class LocalSession extends Session implements Comparable<LocalSession> { Optional<ApplicationSet> currentActiveApplicationSet, Path tenantPath, Instant now) { + applicationRepo.createApplication(params.getApplicationId()); // TODO jvenstad: This is wrong, but it has to be done now, since preparation can change the application ID of a session :( Curator.CompletionWaiter waiter = zooKeeperClient.createPrepareWaiter(); ConfigChangeActions actions = sessionPreparer.prepare(sessionContext, logger, params, currentActiveApplicationSet, tenantPath, now); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java index 61f5e4f1230..c45513e3a07 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSession.java @@ -5,8 +5,8 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.TenantName; import com.yahoo.lang.SettableOptional; -import com.yahoo.transaction.Transaction; import com.yahoo.log.LogLevel; +import com.yahoo.transaction.Transaction; import com.yahoo.vespa.config.server.GlobalComponentRegistry; import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.application.ApplicationSet; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java index ccd5684b9ff..bbb06515bef 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/RemoteSessionRepo.java @@ -1,33 +1,36 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import java.time.Duration; -import java.time.Instant; -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - import com.google.common.collect.HashMultiset; import com.google.common.collect.Multiset; import com.yahoo.concurrent.ThreadFactoryFactory; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.log.LogLevel; import com.yahoo.path.Path; -import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.ReloadHandler; import com.yahoo.vespa.config.server.application.TenantApplications; +import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.tenant.TenantRepository; +import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; import com.yahoo.vespa.curator.Curator; import com.yahoo.yolean.Exceptions; -import com.yahoo.vespa.config.server.ReloadHandler; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.vespa.config.server.monitoring.MetricUpdater; -import com.yahoo.vespa.config.server.zookeeper.ConfigCurator; - import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.*; +import org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Will watch/prepare sessions (applications) based on watched nodes in ZooKeeper, set for example @@ -115,19 +118,6 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { return (created.plus(expiryTime).isBefore(Instant.now())); } - private void loadActiveSession(RemoteSession session) { - tryReload(session.ensureApplicationLoaded(), session.logPre()); - } - - private void tryReload(ApplicationSet applicationSet, String logPre) { - try { - reloadHandler.reloadConfig(applicationSet); - log.log(LogLevel.INFO, logPre + "Application activated successfully: " + applicationSet.getId()); - } catch (Exception e) { - log.log(LogLevel.WARNING, logPre + "Skipping loading of application '" + applicationSet.getId() + "': " + Exceptions.toMessageString(e)); - } - } - private List<Long> getSessionListFromDirectoryCache(List<ChildData> children) { return getSessionList(children.stream() .map(child -> Path.fromString(child.getPath()).getName()) @@ -194,11 +184,12 @@ public class RemoteSessionRepo extends SessionRepo<RemoteSession> { try { if (applicationRepo.requireActiveSessionOf(applicationId) == session.getSessionId()) { log.log(LogLevel.DEBUG, "Found active application for session " + session.getSessionId() + " , loading it"); - loadActiveSession(session); - break; + reloadHandler.reloadConfig(session.ensureApplicationLoaded()); + log.log(LogLevel.INFO, session.logPre() + "Application activated successfully: " + applicationId); + return; } } catch (Exception e) { - log.log(LogLevel.WARNING, session.logPre() + " error reading session id for " + applicationId, e); + log.log(LogLevel.WARNING, session.logPre() + "Skipping loading of application '" + applicationId + "': " + Exceptions.toMessageString(e)); } } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 43a5ff6d0c2..f0ceeb186fe 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -113,9 +113,8 @@ public class SessionPreparer { log.log(LogLevel.DEBUG, () -> "time used " + params.getTimeoutBudget().timesUsed() + " : " + params.getApplicationId()); return preparation.result(); - } catch (OutOfCapacityException e) { - throw e; - } catch (IllegalArgumentException e) { + } + catch (IllegalArgumentException e) { throw new InvalidApplicationException("Invalid application package", e); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java index 078b6e861a9..943ae6248fc 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantBuilder.java @@ -31,7 +31,7 @@ public class TenantBuilder { private SessionFactory sessionFactory; private LocalSessionLoader localSessionLoader; private TenantApplications applicationRepo; - private ReloadHandler reloadHandler; + private TenantRequestHandler reloadHandler; private RequestHandler requestHandler; private RemoteSessionFactory remoteSessionFactory; private TenantFileSystemDirs tenantFileSystemDirs; @@ -120,7 +120,7 @@ public class TenantBuilder { private void createApplicationRepo() { if (applicationRepo == null) { - applicationRepo = TenantApplications.create(componentRegistry.getCurator(), reloadHandler, tenant); + applicationRepo = reloadHandler.applications(); } } @@ -130,7 +130,8 @@ public class TenantBuilder { tenant, Collections.singletonList(componentRegistry.getReloadListener()), ConfigResponseFactory.create(componentRegistry.getConfigserverConfig()), - componentRegistry.getHostRegistries()); + componentRegistry.getHostRegistries(), + componentRegistry.getCurator()); if (hostValidator == null) { this.hostValidator = impl; } @@ -164,9 +165,5 @@ public class TenantBuilder { } } - public TenantApplications getApplicationRepo() { - return applicationRepo; - } - public TenantName getTenantName() { return tenant; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java index f72adcd94dc..fe34e6c361d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandler.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.server.NotFoundException; import com.yahoo.vespa.config.server.application.ApplicationMapper; import com.yahoo.vespa.config.server.application.ApplicationSet; +import com.yahoo.vespa.config.server.application.TenantApplications; import com.yahoo.vespa.config.server.rpc.ConfigResponseFactory; import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.host.HostRegistry; @@ -29,6 +30,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.curator.Curator; /** * A per tenant request handler, for handling reload (activate application) and getConfig requests for @@ -44,23 +46,25 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host private final TenantName tenant; private final List<ReloadListener> reloadListeners; private final ConfigResponseFactory responseFactory; - private final HostRegistry<ApplicationId> hostRegistry; private final ApplicationMapper applicationMapper = new ApplicationMapper(); private final MetricUpdater tenantMetricUpdater; private final Clock clock = Clock.systemUTC(); + private final TenantApplications applications; public TenantRequestHandler(Metrics metrics, TenantName tenant, List<ReloadListener> reloadListeners, ConfigResponseFactory responseFactory, - HostRegistries hostRegistries) { + HostRegistries hostRegistries, + Curator curator) { // TODO jvenstad: Merge this class with TenantApplications, and straighten this out. this.metrics = metrics; this.tenant = tenant; - this.reloadListeners = reloadListeners; + this.reloadListeners = List.copyOf(reloadListeners); this.responseFactory = responseFactory; - tenantMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenant)); - hostRegistry = hostRegistries.createApplicationHostRegistry(tenant); + this.tenantMetricUpdater = metrics.getOrCreateMetricUpdater(Metrics.createDimensions(tenant)); + this.hostRegistry = hostRegistries.createApplicationHostRegistry(tenant); + this.applications = TenantApplications.create(curator, this, tenant); } /** @@ -93,6 +97,7 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host * * @param applicationSet the {@link ApplicationSet} to be reloaded */ + @Override public void reloadConfig(ApplicationSet applicationSet) { setLiveApp(applicationSet); notifyReloadListeners(applicationSet); @@ -112,7 +117,7 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host @Override public void removeApplicationsExcept(Set<ApplicationId> applications) { for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) { - if (! applications.contains(activeApplication)) { + if ( ! applications.contains(activeApplication)) { log.log(LogLevel.INFO, "Will remove deleted application " + activeApplication.toShortString()); removeApplication(activeApplication); } @@ -237,4 +242,6 @@ public class TenantRequestHandler implements RequestHandler, ReloadHandler, Host } } + TenantApplications applications() { return applications; } + } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index 01a7d5e0239..69c88dc0275 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -41,10 +41,10 @@ public class TenantApplicationsTest { TenantApplications repo = createZKAppRepo(); List<ApplicationId> applications = repo.activeApplications(); assertThat(applications.size(), is(2)); - assertThat(applications.get(0).application().value(), is("foo")); - assertThat(applications.get(1).application().value(), is("bar")); - assertThat(repo.requireActiveSessionOf(applications.get(0)), is(3L)); - assertThat(repo.requireActiveSessionOf(applications.get(1)), is(4L)); + assertThat(applications.get(0).application().value(), is("bar")); + assertThat(applications.get(1).application().value(), is("foo")); + assertThat(repo.requireActiveSessionOf(applications.get(0)), is(4L)); + assertThat(repo.requireActiveSessionOf(applications.get(1)), is(3L)); } @Test @@ -77,6 +77,7 @@ public class TenantApplicationsTest { public void require_that_application_ids_can_be_written() throws Exception { TenantApplications repo = createZKAppRepo(); ApplicationId myapp = createApplicationId("myapp"); + repo.createApplication(myapp); repo.createPutTransaction(myapp, 3l).commit(); String path = TenantRepository.getApplicationsPath(tenantName).append(myapp.serializedForm()).getAbsolute(); assertTrue(curatorFramework.checkExists().forPath(path) != null); @@ -91,6 +92,8 @@ public class TenantApplicationsTest { TenantApplications repo = createZKAppRepo(); ApplicationId id1 = createApplicationId("myapp"); ApplicationId id2 = createApplicationId("myapp2"); + repo.createApplication(id1); + repo.createApplication(id2); repo.createPutTransaction(id1, 1).commit(); repo.createPutTransaction(id2, 1).commit(); assertThat(repo.activeApplications().size(), is(2)); @@ -126,11 +129,7 @@ public class TenantApplicationsTest { } private static ApplicationId createApplicationId(String name) { - return new ApplicationId.Builder() - .tenant(tenantName.value()) - .applicationName(name) - .instanceName("myinst") - .build(); + return ApplicationId.from(tenantName.value(), name, "myinst"); } private void writeApplicationData(ApplicationId applicationId, long sessionId) throws Exception { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index 3d34a4eeaf5..c6a8e1f2f9d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -56,11 +56,13 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { session2 = new MockSession(2l, FilesApplicationPackage.fromFile(new File("src/test/apps/content"))); Tenant tenant1 = tenantRepository.getTenant(tenantName1); tenant1.getLocalSessionRepo().addSession(session2); + tenant1.getApplicationRepo().createApplication(idTenant1); tenant1.getApplicationRepo().createPutTransaction(idTenant1, 2l).commit(); MockSession session3 = new MockSession(3l, FilesApplicationPackage.fromFile(new File("src/test/apps/content2"))); Tenant tenant2 = tenantRepository.getTenant(tenantName2); tenant2.getLocalSessionRepo().addSession(session3); + tenant2.getApplicationRepo().createApplication(idTenant2); tenant2.getApplicationRepo().createPutTransaction(idTenant2, 3l).commit(); handler = new ApplicationHandler(ApplicationHandler.testOnlyContext(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index fb75e91dfd6..1db70956407 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -46,6 +46,7 @@ public class HostHandlerTest { private HostHandler hostHandler; static void addMockApplication(Tenant tenant, ApplicationId applicationId, long sessionId) { + tenant.getApplicationRepo().createApplication(applicationId); tenant.getApplicationRepo().createPutTransaction(applicationId, sessionId).commit(); ApplicationPackage app = FilesApplicationPackage.fromFile(testApp); tenant.getLocalSessionRepo().addSession(new SessionHandlerTest.MockSession(sessionId, app, applicationId)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java index b34f7a0c487..f97bc443a38 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ListApplicationsHandlerTest.java @@ -51,17 +51,17 @@ public class ListApplicationsHandlerTest { final String url = "http://myhost:14000/application/v2/tenant/mytenant/application/"; assertResponse(url, Response.Status.OK, "[]"); - applicationRepo.createPutTransaction( - new ApplicationId.Builder().tenant("tenant").applicationName("foo").instanceName("quux").build(), - 1).commit(); + ApplicationId id1 = ApplicationId.from("mytenant", "foo", "quux"); + applicationRepo.createApplication(id1); + applicationRepo.createPutTransaction(id1, 1).commit(); assertResponse(url, Response.Status.OK, "[\"" + url + "foo/environment/dev/region/us-east/instance/quux\"]"); - applicationRepo.createPutTransaction( - new ApplicationId.Builder().tenant("tenant").applicationName("bali").instanceName("quux").build(), - 1).commit(); + ApplicationId id2 = ApplicationId.from("mytenant", "bali", "quux"); + applicationRepo.createApplication(id2); + applicationRepo.createPutTransaction(id2, 1).commit(); assertResponse(url, Response.Status.OK, - "[\"" + url + "foo/environment/dev/region/us-east/instance/quux\"," + - "\"" + url + "bali/environment/dev/region/us-east/instance/quux\"]" + "[\"" + url + "bali/environment/dev/region/us-east/instance/quux\"," + + "\"" + url + "foo/environment/dev/region/us-east/instance/quux\"]" ); } @@ -82,12 +82,12 @@ public class ListApplicationsHandlerTest { @Test public void require_that_listing_works_with_multiple_tenants() throws Exception { - applicationRepo.createPutTransaction(new ApplicationId.Builder() - .tenant("tenant") - .applicationName("foo").instanceName("quux").build(), 1).commit(); - applicationRepo2.createPutTransaction(new ApplicationId.Builder() - .tenant("tenant") - .applicationName("quux").instanceName("foo").build(), 1).commit(); + ApplicationId id1 = ApplicationId.from("mytenant", "foo", "quux"); + applicationRepo.createApplication(id1); + applicationRepo.createPutTransaction(id1, 1).commit(); + ApplicationId id2 = ApplicationId.from("foobar", "quux", "foo"); + applicationRepo2.createApplication(id2); + applicationRepo2.createPutTransaction(id2, 1).commit(); String url = "http://myhost:14000/application/v2/tenant/mytenant/application/"; assertResponse(url, Response.Status.OK, "[\"" + url + "foo/environment/dev/region/us-east/instance/quux\"]"); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 380b76c30af..858d1e0eaa7 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -323,6 +323,9 @@ public class SessionActiveHandlerTest extends SessionHandlerTest { Optional.of(AllocatedHosts.withHosts(Collections.singleton(new HostSpec("bar", Collections.emptyList()))))); session = createRemoteSession(sessionId, initialStatus, zkClient); addLocalSession(sessionId, deployData, zkClient); + tenantRepository.getTenant(tenantName).getApplicationRepo().createApplication(ApplicationId.from(tenantName.value(), + deployData.getApplicationName(), + InstanceName.defaultName().value())); metaData = localRepo.getSession(sessionId).getMetaData(); actResponse = handler.handle(SessionHandlerTest.createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.ACTIVE, sessionId, subPath)); return this; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index bc509fcd802..0946ef3992c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -194,6 +194,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { .applicationName("foo") .instanceName("quux") .build(); + applicationRepo.createApplication(fooId); applicationRepo.createPutTransaction(fooId, 2).commit(); assertFromParameter("3", "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foo/environment/test/region/baz/instance/quux"); localSessionRepo.addSession(new SessionHandlerTest.MockSession(5l, FilesApplicationPackage.fromFile(testApp))); @@ -202,6 +203,7 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { .applicationName("foobio") .instanceName("quux") .build(); + applicationRepo.createApplication(bioId); applicationRepo.createPutTransaction(bioId, 5).commit(); assertFromParameter("6", "http://myhost:40555/application/v2/tenant/" + tenant + "/application/foobio/environment/staging/region/baz/instance/quux"); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java index e7db4dcf58f..a4432dcbfcd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/LocalSessionTest.java @@ -196,12 +196,14 @@ public class LocalSessionTest { zkClient.write(Collections.singletonMap(new Version(0, 0, 0), new MockFileRegistry())); File sessionDir = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); sessionDir.createNewFile(); + TenantApplications applications = TenantApplications.create(curator, new MockReloadHandler(), tenant); + applications.createApplication(zkc.readApplicationId()); return new LocalSession(tenant, sessionId, preparer, new SessionContext( FilesApplicationPackage.fromFile(testApp), zkc, sessionDir, - TenantApplications.create(curator, new MockReloadHandler(), tenant), + applications, new HostRegistry<>(), superModelGenerationCounter, flagSource)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java index a7b74e69a21..d5d0fe72dbe 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/RemoteSessionTest.java @@ -3,23 +3,26 @@ package com.yahoo.vespa.config.server.session; import com.google.common.io.Files; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.api.*; +import com.yahoo.config.model.api.Model; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.api.ModelCreateResult; +import com.yahoo.config.model.api.ModelFactory; +import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.component.Version; -import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.TestComponentRegistry; +import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.PermanentApplicationPackage; -import com.yahoo.vespa.curator.mock.MockCurator; +import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.VespaModelFactory; - import org.junit.Before; import org.junit.Test; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java index 5f18046cb81..952f87cbc6d 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRequestHandlerTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.tenant; +import com.yahoo.component.Version; import com.yahoo.config.ConfigInstance; import com.yahoo.config.SimpletypesConfig; import com.yahoo.config.application.api.ApplicationPackage; @@ -9,9 +10,10 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.DeployData; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AllocatedHosts; -import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.TenantName; import com.yahoo.io.IOUtils; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayload; @@ -19,27 +21,24 @@ import com.yahoo.vespa.config.GetConfigRequest; import com.yahoo.vespa.config.protocol.ConfigResponse; import com.yahoo.vespa.config.protocol.DefContent; import com.yahoo.vespa.config.protocol.VespaVersion; -import com.yahoo.vespa.config.server.application.ApplicationSet; -import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.ReloadListener; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.TestComponentRegistry; -import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory; import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; +import com.yahoo.vespa.config.server.host.HostRegistries; import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; import com.yahoo.vespa.config.server.monitoring.MetricUpdater; import com.yahoo.vespa.config.server.monitoring.Metrics; +import com.yahoo.vespa.config.server.rpc.UncompressedConfigResponseFactory; import com.yahoo.vespa.config.server.session.RemoteSession; import com.yahoo.vespa.config.server.session.SessionZooKeeperClient; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.VespaModelFactory; - import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -48,11 +47,22 @@ import org.xml.sax.SAXException; import java.io.File; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import static org.hamcrest.core.Is.is; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen @@ -83,7 +93,7 @@ public class TenantRequestHandlerTest { Metrics sh = Metrics.createTestMetrics(); List<ReloadListener> listeners = new ArrayList<>(); listeners.add(listener); - server = new TenantRequestHandler(sh, tenant, listeners, new UncompressedConfigResponseFactory(), new HostRegistries()); + server = new TenantRequestHandler(sh, tenant, listeners, new UncompressedConfigResponseFactory(), new HostRegistries(), curator); componentRegistry = new TestComponentRegistry.Builder() .curator(curator) .modelFactoryRegistry(createRegistry()) diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 885b157b4c3..b968d1c391f 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -5066,6 +5066,7 @@ "public void addItem(com.yahoo.prelude.query.Item)", "public void addItem(int, com.yahoo.prelude.query.Item)", "public boolean isEmpty()", + "public com.yahoo.prelude.query.Item and(com.yahoo.prelude.query.Item)", "public static java.util.List getPositiveTerms(com.yahoo.prelude.query.Item)", "public bridge synthetic com.yahoo.prelude.query.CompositeItem clone()", "public bridge synthetic com.yahoo.prelude.query.Item clone()", diff --git a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java index bacfe8a949a..6eba6ae4837 100644 --- a/container-search/src/main/java/com/yahoo/search/query/QueryTree.java +++ b/container-search/src/main/java/com/yahoo/search/query/QueryTree.java @@ -108,9 +108,12 @@ public class QueryTree extends CompositeItem { // -------------- Facade - /** Modifies this query to become the current query AND the given item */ - // TODO: Make sure this is complete, unit test and make it public - private void and(Item item) { + /** + * Modifies this query to become the current query AND the given item. + * + * @return the resulting root item in this + */ + public Item and(Item item) { if (isEmpty()) { setRoot(item); } @@ -126,12 +129,16 @@ public class QueryTree extends CompositeItem { notItem.addPositiveItem(getRoot()); setRoot(notItem); } + else if (getRoot() instanceof AndItem) { + ((AndItem) getRoot()).addItem(item); + } else { AndItem andItem = new AndItem(); andItem.addItem(getRoot()); andItem.addItem(item); setRoot(andItem); } + return getRoot(); } /** Returns a flattened list of all positive query terms under the given item */ diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java index 21bc22d7002..f77301f587c 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java @@ -20,7 +20,8 @@ import static com.yahoo.yolean.Exceptions.toMessageString; /** * Searcher that builds a PredicateItem from the &boolean properties and inserts it into a query. - * @author <a href="mailto:magnarn@yahoo-inc.com">Magnar Nedland</a> + * + * @author Magnar Nedland */ @After({ STEMMING, ACCENT_REMOVAL }) @Provides(BooleanSearcher.PREDICATE) @@ -74,7 +75,7 @@ public class BooleanSearcher extends Searcher { item.setIndexName(fieldName); new PredicateValueAttributeParser(item).parse(attributes); new PredicateRangeAttributeParser(item).parse(rangeAttributes); - QueryTreeUtil.andQueryItemWithRoot(query, item); + query.getModel().getQueryTree().and(item); } static public class PredicateValueAttributeParser extends BooleanAttributeParser { diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java b/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java index e4841ae6bd1..759c8ba1ee4 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/QueryTreeUtil.java @@ -3,6 +3,7 @@ package com.yahoo.search.querytransform; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.QueryCanonicalizer; import com.yahoo.search.Query; import com.yahoo.search.query.QueryTree; @@ -10,27 +11,27 @@ import com.yahoo.search.query.QueryTree; * Utility class for manipulating a QueryTree. * * @author geirst + * @deprecated use QueryTree.and instead // TODO: Remove on Vespa 8 */ +@Deprecated public class QueryTreeUtil { - static public void andQueryItemWithRoot(Query query, Item item) { - andQueryItemWithRoot(query.getModel().getQueryTree(), item); + /** + * Adds the given item to this query + * + * @return the new root of the query tree + */ + static public Item andQueryItemWithRoot(Query query, Item item) { + return andQueryItemWithRoot(query.getModel().getQueryTree(), item); } - static public void andQueryItemWithRoot(QueryTree tree, Item item) { - if (tree.isEmpty()) { - tree.setRoot(item); - } else { - Item oldRoot = tree.getRoot(); - if (oldRoot.getClass() == AndItem.class) { - ((AndItem) oldRoot).addItem(item); - } else { - AndItem newRoot = new AndItem(); - newRoot.addItem(oldRoot); - newRoot.addItem(item); - tree.setRoot(newRoot); - } - } + /** + * Adds the given item to this query + * + * @return the new root of the query tree + */ + static public Item andQueryItemWithRoot(QueryTree tree, Item item) { + return tree.and(item); } } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java index cda41f5f62e..0b1387a16a2 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java @@ -15,7 +15,6 @@ import com.yahoo.text.MapParser; import java.util.LinkedHashMap; import java.util.Map; -import static com.yahoo.container.protect.Error.UNSPECIFIED; import com.yahoo.yolean.Exceptions; /** @@ -147,7 +146,7 @@ public class WandSearcher extends Searcher { InputResolver inputs = new InputResolver(query, execution); if ( ! inputs.hasValidData()) return execution.search(query); - QueryTreeUtil.andQueryItemWithRoot(query, createWandQueryItem(inputs)); + query.getModel().getQueryTree().and(createWandQueryItem(inputs)); query.trace("WandSearcher: Added WAND operator", true, 4); return execution.search(query); } diff --git a/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java b/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java new file mode 100644 index 00000000000..f929e54fd2d --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/query/QueryTreeTest.java @@ -0,0 +1,26 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.query; + +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.WordItem; +import org.junit.Assert; +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +/** + * @author bratseth + */ +public class QueryTreeTest { + + @Test + public void testAddQueryItemWithRoot() { + Assert.assertEquals("AND a b", + new QueryTree(new WordItem("a")).and(new WordItem("b")).toString()); + + NotItem not = new NotItem(); + not.addNegativeItem(new WordItem("b")); + assertEquals("+a -b", + new QueryTree(new WordItem("a")).and(not).toString()); + } + +} diff --git a/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java b/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java index c7a44e8aceb..720dcd0c4bc 100644 --- a/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java +++ b/container-search/src/test/java/com/yahoo/search/querytransform/TestUtils.java @@ -6,7 +6,9 @@ import com.yahoo.prelude.query.Item; import com.yahoo.search.Result; public class TestUtils { + public static Item getQueryTreeRoot(Result result) { return result.getQuery().getModel().getQueryTree().getRoot(); } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java index aba3b5f3ab7..fffa849f7d3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactory.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.athenz; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.athenz.client.zts.ZtsClient; @@ -10,7 +11,7 @@ import com.yahoo.vespa.athenz.client.zts.ZtsClient; */ public interface AthenzClientFactory { - AthenzService getControllerIdentity(); + AthenzIdentity getControllerIdentity(); ZmsClient createZmsClient(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java index d166bb0d3fb..d618464fc2a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/Node.java @@ -31,10 +31,14 @@ public class Node { private final long wantedRestartGeneration; private final long rebootGeneration; private final long wantedRebootGeneration; + private final String canonicalFlavor; + private final String clusterId; + private final ClusterType clusterType; public Node(HostName hostname, State state, NodeType type, Optional<ApplicationId> owner, Version currentVersion, Version wantedVersion, Version currentOsVersion, Version wantedOsVersion, ServiceState serviceState, - long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration) { + long restartGeneration, long wantedRestartGeneration, long rebootGeneration, long wantedRebootGeneration, + String canonicalFlavor, String clusterId, ClusterType clusterType) { this.hostname = hostname; this.state = state; this.type = type; @@ -48,13 +52,17 @@ public class Node { this.wantedRestartGeneration = wantedRestartGeneration; this.rebootGeneration = rebootGeneration; this.wantedRebootGeneration = wantedRebootGeneration; + this.canonicalFlavor = canonicalFlavor; + this.clusterId = clusterId; + this.clusterType = clusterType; } @TestOnly public Node(HostName hostname, State state, NodeType type, Optional<ApplicationId> owner, Version currentVersion, Version wantedVersion) { this(hostname, state, type, owner, currentVersion, wantedVersion, - Version.emptyVersion, Version.emptyVersion, ServiceState.unorchestrated, 0, 0, 0, 0); + Version.emptyVersion, Version.emptyVersion, ServiceState.unorchestrated, 0, 0, 0, 0, + "d-2-8-50", "cluster", ClusterType.container); } public HostName hostname() { @@ -107,6 +115,18 @@ public class Node { return wantedRebootGeneration; } + public String canonicalFlavor() { + return canonicalFlavor; + } + + public String clusterId() { + return clusterId; + } + + public ClusterType clusterType() { + return clusterType; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -139,4 +159,11 @@ public class Node { unorchestrated } + /** Known cluster types. */ + public enum ClusterType { + admin, + container, + content + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index c168ddf6caf..db9291cd651 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneFilter; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -53,7 +54,7 @@ public interface ZoneRegistry { SystemName system(); /** Return the configserver's Athenz service identity */ - AthenzService getConfigServerAthenzService(ZoneId zoneId); + AthenzIdentity getConfigServerAthenzIdentity(ZoneId zoneId); /** Returns the Vespa upgrade policy to use for zones in this registry */ UpgradePolicy upgradePolicy(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java index 3ba0367a00c..14d8d06d0c6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Context.java @@ -9,7 +9,7 @@ import java.util.Objects; import java.util.Optional; /** - * The context in which a role is valid. + * The context in which a role is valid. This is immutable. * * @author mpolden */ @@ -40,11 +40,6 @@ public class Context { return system; } - /** Returns whether this context is considered limited */ - public boolean limited() { - return tenant.isPresent() || application.isPresent(); - } - /** Returns a context that has no restrictions on tenant or application in given system */ public static Context unlimitedIn(SystemName system) { return new Context(Optional.empty(), Optional.empty(), system); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 797ca10ed3d..23bf8514b9c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -15,6 +15,7 @@ import java.util.Set; * When creating a new API, its paths must be added here and a policy must be declared in {@link Policy}. * * @author mpolden + * @author jonmv */ public enum PathGroup { @@ -32,31 +33,38 @@ public enum PathGroup { /** Paths used for creating tenants with proper access control. */ tenant(Matcher.tenant, + Optional.of("/api"), "/application/v4/tenant/{tenant}"), /** Paths used for user management on the tenant level. */ tenantUsers(Matcher.tenant, + Optional.of("/api"), "/user/v1/tenant/{tenant}"), /** Paths used by tenant administrators. */ tenantInfo(Matcher.tenant, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/"), /** Path for the base application resource. */ application(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}"), /** Paths used for user management on the application level. */ applicationUsers(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/user/v1/tenant/{tenant}/application/{application}"), /** Paths used by application administrators. */ applicationInfo(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/deploying/{*}", "/application/v4/tenant/{tenant}/application/{application}/instance/{*}", + "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{*}", @@ -65,10 +73,12 @@ public enum PathGroup { /** Path used to restart application nodes. */ // TODO move to the above when everyone is on new pipeline. applicationRestart(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{ignored}/restart"), /** Paths used for development deployments. */ developmentDeployment(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/environment/dev/region/{region}/instance/{instance}", "/application/v4/tenant/{tenant}/application/{application}/environment/dev/region/{region}/instance/{instance}/deploy", "/application/v4/tenant/{tenant}/application/{application}/environment/perf/region/{region}/instance/{instance}", @@ -77,6 +87,7 @@ public enum PathGroup { /** Paths used for production deployments. */ productionDeployment(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/environment/prod/region/{region}/instance/{instance}", "/application/v4/tenant/{tenant}/application/{application}/environment/prod/region/{region}/instance/{instance}/deploy", "/application/v4/tenant/{tenant}/application/{application}/environment/test/region/{region}/instance/{instance}", @@ -87,21 +98,26 @@ public enum PathGroup { /** Paths used for continuous deployment to production. */ submission(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/submit"), /** Paths used for other tasks by build services. */ // TODO: This will vanish. buildService(Matcher.tenant, Matcher.application, + Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/jobreport", "/application/v4/tenant/{tenant}/application/{application}/promote", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/promote"), + /** Paths which contain (not very strictly) classified information about customers. */ + classifiedTenantInfo(Optional.of("/api"), + "/application/v4/", + "/application/v4/tenant/"), + /** Paths which contain (not very strictly) classified information about, e.g., customers. */ classifiedInfo("/athenz/v1/{*}", "/cost/v1/{*}", "/deployment/v1/{*}", - "/application/v4/", - "/application/v4/tenant/", "/", "/d/{*}", "/statuspage/v1/{*}"), @@ -111,30 +127,43 @@ public enum PathGroup { "/zone/v1/{*}"); final List<String> pathSpecs; + final String prefix; final List<Matcher> matchers; PathGroup(String... pathSpecs) { - this(List.of(), List.of(pathSpecs)); + this(List.of(), Optional.empty(), List.of(pathSpecs)); + } + + PathGroup(Optional<String> prefix, String... pathSpecs) { + this(List.of(), prefix, List.of(pathSpecs)); } PathGroup(Matcher first, String... pathSpecs) { - this(List.of(first), List.of(pathSpecs)); + this(List.of(first), Optional.empty(), List.of(pathSpecs)); + } + + PathGroup(Matcher first, Optional<String> prefix, String... pathSpecs) { + this(List.of(first), prefix, List.of(pathSpecs)); } PathGroup(Matcher first, Matcher second, String... pathSpecs) { - this(List.of(first, second), List.of(pathSpecs)); + this(List.of(first, second), Optional.empty(), List.of(pathSpecs)); + } + + PathGroup(Matcher first, Matcher second, Optional<String> prefix, String... pathSpecs) { + this(List.of(first, second), prefix, List.of(pathSpecs)); } /** Creates a new path group, if the given context matchers are each present exactly once in each of the given specs. */ - PathGroup(List<Matcher> matchers, List<String> pathSpecs) { + PathGroup(List<Matcher> matchers, Optional<String> prefix, List<String> pathSpecs) { this.matchers = matchers; + this.prefix = prefix.orElse(""); this.pathSpecs = pathSpecs; } /** Returns path if it matches any spec in this group, with match groups set by the match. */ - @SuppressWarnings("deprecation") private Optional<Path> get(URI uri) { - Path matcher = new Path(uri); // TODO Get URI down here. + Path matcher = new Path(uri, prefix); for (String spec : pathSpecs) // Iterate to be sure the Path's state is that of the match. if (matcher.matches(spec)) return Optional.of(matcher); return Optional.empty(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java index ff535e92033..c28fa7a3fc3 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/Role.java @@ -4,8 +4,6 @@ package com.yahoo.vespa.hosted.controller.api.role; import java.net.URI; import java.util.Objects; -import static java.util.Objects.requireNonNull; - /** * A role is a combination of a {@link RoleDefinition} and a {@link Context}, which allows evaluation * of access control for a given action on a resource. Create using {@link Roles}. @@ -18,15 +16,15 @@ public abstract class Role { final Context context; Role(RoleDefinition roleDefinition, Context context) { - this.roleDefinition = requireNonNull(roleDefinition); - this.context = requireNonNull(context); + this.roleDefinition = Objects.requireNonNull(roleDefinition); + this.context = Objects.requireNonNull(context); } /** Returns the role definition of this bound role. */ public RoleDefinition definition() { return roleDefinition; } /** Returns whether this role is allowed to perform the given action on the given resource. */ - public boolean allows(Action action, URI uri) { + public final boolean allows(Action action, URI uri) { return roleDefinition.policies().stream().anyMatch(policy -> policy.evaluate(action, uri, context)); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java index 41444258a68..3378f9e0061 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SecurityContext.java @@ -5,8 +5,9 @@ import java.security.Principal; import java.util.Objects; import java.util.Set; -import static java.util.Objects.requireNonNull; - +/** + * @author tokle + */ public class SecurityContext { public static final String ATTRIBUTE_NAME = SecurityContext.class.getName(); @@ -15,7 +16,7 @@ public class SecurityContext { private final Set<Role> roles; public SecurityContext(Principal principal, Set<Role> roles) { - this.principal = requireNonNull(principal); + this.principal = Objects.requireNonNull(principal); this.roles = Set.copyOf(roles); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 3664f3712e1..84e15deea4c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -9,16 +9,16 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationActivity; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -199,9 +199,10 @@ public class Application { return rotation; } - /** Returns the global rotation dns name, if present */ - public Optional<GlobalDnsName> globalDnsName(SystemName system) { - return rotation.map(ignored -> new GlobalDnsName(id, system)); + /** Returns the default global endpoints for this in given system */ + public EndpointList endpointsIn(SystemName system) { + if (rotation.isEmpty()) return EndpointList.EMPTY; + return EndpointList.defaultGlobal(id, system); } public Optional<String> pemDeployKey() { return pemDeployKey; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 7caefa55d28..4c5be570a02 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -45,7 +45,8 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; @@ -283,7 +284,7 @@ public class ApplicationController { ApplicationVersion applicationVersion; ApplicationPackage applicationPackage; Set<String> rotationNames = new HashSet<>(); - Set<String> cnames = new HashSet<>(); + Set<String> cnames; try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(require(applicationId), lock); @@ -324,13 +325,7 @@ public class ApplicationController { // Assign global rotation application = withRotation(application, zone); Application app = application.get(); - app.globalDnsName(controller.system()).ifPresent(applicationRotation -> { - rotationNames.add(app.rotation().orElseThrow(() -> new RuntimeException("Global Dns assigned, but no rotation id present")).asString()); - cnames.add(applicationRotation.dnsName()); - cnames.add(applicationRotation.secureDnsName()); - cnames.add(applicationRotation.oathDnsName()); - }); - + cnames = app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).collect(Collectors.toSet()); // Update application with information from application package if ( ! preferOldestVersion && ! application.get().deploymentJobs().deployedInternally() @@ -438,18 +433,20 @@ public class ApplicationController { application = application.with(rotation.id()); store(application); // store assigned rotation even if deployment fails - GlobalDnsName dnsName = application.get().globalDnsName(controller.system()) - .orElseThrow(() -> new IllegalStateException("Expected rotation to be assigned")); boolean redirectLegacyDns = redirectLegacyDnsFlag.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()) .value(); - registerCname(dnsName.oathDnsName(), rotation.name()); - if (redirectLegacyDns) { - registerCname(dnsName.dnsName(), dnsName.oathDnsName()); - registerCname(dnsName.secureDnsName(), dnsName.oathDnsName()); - } else { - registerCname(dnsName.dnsName(), rotation.name()); - registerCname(dnsName.secureDnsName(), rotation.name()); - } + + EndpointList globalEndpoints = application.get() + .endpointsIn(controller.system()) + .scope(Endpoint.Scope.global); + globalEndpoints.main().ifPresent(mainEndpoint -> { + registerCname(mainEndpoint.dnsName(), rotation.name()); + if (redirectLegacyDns) { + globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), mainEndpoint.dnsName())); + } else { + globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), rotation.name())); + } + }); } } return application; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java new file mode 100644 index 00000000000..14a5d3c7ddf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -0,0 +1,281 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; + +import java.net.URI; +import java.util.Objects; + +/** + * Represents an application's endpoint. The endpoint scope can either be global or a specific zone. This is visible to + * the tenant and is used by the tenant when accessing deployments. + * + * @author mpolden + */ +public class Endpoint { + + public static final String YAHOO_DNS_SUFFIX = ".vespa.yahooapis.com"; + public static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud"; + public static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud"; + + private final URI url; + private final Scope scope; + private final boolean legacy; + private final boolean directRouting; + + private Endpoint(String name, ApplicationId application, ZoneId zone, SystemName system, Port port, boolean legacy, + boolean directRouting) { + Objects.requireNonNull(name, "name must be non-null"); + Objects.requireNonNull(application, "application must be non-null"); + Objects.requireNonNull(system, "system must be non-null"); + Objects.requireNonNull(port, "port must be non-null"); + this.url = createUrl(name, application, zone, system, port, legacy, directRouting); + this.scope = zone == null ? Scope.global : Scope.zone; + this.legacy = legacy; + this.directRouting = directRouting; + } + + /** Returns the URL used to access this */ + public URI url() { + return url; + } + + /** Returns the DNS name of this */ + public String dnsName() { + return url.getHost(); + } + + /** Returns the scope of this */ + public Scope scope() { + return scope; + } + + /** Returns whether this is considered a legacy DNS name that is due for removal */ + public boolean legacy() { + return legacy; + } + + /** + * Returns whether this endpoint supports direct routing. Direct routing means that this endpoint is served by an + * exclusive load balancer instead of a shared routing layer. + */ + public boolean directRouting() { + return directRouting; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Endpoint endpoint = (Endpoint) o; + return url.equals(endpoint.url); + } + + @Override + public int hashCode() { + return Objects.hash(url); + } + + @Override + public String toString() { + return String.format("endpoint %s [scope=%s, legacy=%s, directRouting=%s]", url, scope, legacy, directRouting); + } + + private static URI createUrl(String name, ApplicationId application, ZoneId zone, SystemName system, + Port port, boolean legacy, boolean directRouting) { + String scheme = port.tls ? "https" : "http"; + String separator = separator(system, directRouting, port.tls); + String portPart = port.isDefault() ? "" : ":" + port.port; + return URI.create(scheme + "://" + + sanitize(namePart(name, separator)) + + systemPart(system, separator) + + sanitize(instancePart(application, zone, separator)) + + sanitize(application.application().value()) + + separator + + sanitize(application.tenant().value()) + + "." + + scopePart(zone, legacy) + + dnsSuffix(system, legacy) + + portPart + + "/"); + } + + private static String sanitize(String part) { // TODO: Reject reserved words + return part.replace('_', '-'); + } + + private static String separator(SystemName system, boolean directRouting, boolean tls) { + if (!tls) return "."; + if (directRouting) return "."; + if (isPublic(system)) return "."; + return "--"; + } + + private static String namePart(String name, String separator) { + if ("default".equals(name)) return ""; + return name + separator; + } + + private static String scopePart(ZoneId zone, boolean legacy) { + if (zone == null) return "global"; + if (!legacy && zone.environment().isProduction()) return zone.region().value(); // Skip prod environment for non-legacy endpoints + return zone.region().value() + "." + zone.environment().value(); + } + + private static String instancePart(ApplicationId application, ZoneId zone, String separator) { + if (zone == null) return ""; // Always omit instance for global endpoints + if (application.instance().isDefault()) return ""; // Skip "default" + return application.instance().value() + separator; + } + + private static String systemPart(SystemName system, String separator) { + if (system == SystemName.main || isPublic(system)) return ""; + return system.name() + separator; + } + + private static String dnsSuffix(SystemName system, boolean legacy) { + switch (system) { + case cd: + case main: + if (legacy) return YAHOO_DNS_SUFFIX; + return OATH_DNS_SUFFIX; + case Public: + case vaas: + return PUBLIC_DNS_SUFFIX; + default: throw new IllegalArgumentException("No DNS suffix declared for system " + system); + } + } + + private static boolean isPublic(SystemName system) { // TODO: Remove and inline once we're down to one + return system == SystemName.Public || system == SystemName.vaas; + } + + /** An endpoint's scope */ + public enum Scope { + + /** Endpoint points to all zones */ + global, + + /** Endpoint points to a single zone */ + zone, + + } + + /** Represents an endpoint's HTTP port */ + public static class Port { + + private final int port; + private final boolean tls; + + private Port(int port, boolean tls) { + if (port < 1 || port > 65535) { + throw new IllegalArgumentException("Port must be between 1 and 65535, got " + port); + } + this.port = port; + this.tls = tls; + } + + private boolean isDefault() { + return port == 80 || port == 443; + } + + /** Returns the default HTTPS port */ + public static Port tls() { + return new Port(443, true); + } + + /** Create a HTTPS port */ + public static Port tls(int port) { + return new Port(port, true); + } + + /** Create a HTTP port */ + public static Port plain(int port) { + return new Port(port, false); + } + + } + + /** Build an endpoint for given application */ + public static EndpointBuilder of(ApplicationId application) { + return new EndpointBuilder(application); + } + + public static class EndpointBuilder { + + private final ApplicationId application; + + private ZoneId zone; + private ClusterSpec.Id cluster; + private RotationName rotation; + private Port port; + private boolean legacy = false; + private boolean directRouting = false; + + private EndpointBuilder(ApplicationId application) { + this.application = application; + } + + /** Sets the cluster and zone target of this */ + public EndpointBuilder target(ClusterSpec.Id cluster, ZoneId zone) { + if (rotation != null) { + throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + } + this.cluster = cluster; + this.zone = zone; + return this; + } + + /** Sets the rotation target of this */ + public EndpointBuilder target(RotationName rotation) { + if (cluster != null && zone != null) { + throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + } + this.rotation = rotation; + return this; + } + + /** Sets the port of this */ + public EndpointBuilder on(Port port) { + this.port = port; + return this; + } + + /** Marks this as a legacy endpoint */ + public EndpointBuilder legacy() { + this.legacy = true; + return this; + } + + /** Enables direct routing support for this */ + public EndpointBuilder directRouting() { + this.directRouting = true; + return this; + } + + /** Sets the system that owns this */ + public Endpoint in(SystemName system) { + String name; + if (cluster != null && zone != null) { + name = cluster.value(); + } else if (rotation != null) { + name = rotation.value(); + } else { + throw new IllegalArgumentException("Must set either cluster or rotation target"); + } + if (isPublic(system) && !directRouting) { + throw new IllegalArgumentException("Public system only supports direct routing endpoints"); + } + if (directRouting && !port.isDefault()) { + throw new IllegalArgumentException("Direct routing endpoints only support default port"); + } + return new Endpoint(name, application, zone, system, port, legacy, directRouting); + } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java new file mode 100644 index 00000000000..0c04a1f099c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -0,0 +1,85 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; + +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +/** + * A list of endpoints for an application. + * + * @author mpolden + */ +public class EndpointList { + + public static final EndpointList EMPTY = new EndpointList(List.of()); + + private final List<Endpoint> endpoints; + + private EndpointList(List<Endpoint> endpoints) { + long mainEndpoints = endpoints.stream() + .filter(endpoint -> endpoint.scope() == Endpoint.Scope.global) + .filter(Predicate.not(Endpoint::directRouting)) + .filter(Predicate.not(Endpoint::legacy)).count(); + if (mainEndpoints > 1) { + throw new IllegalArgumentException("Can have only 1 non-legacy global endpoint, got " + endpoints); + } + if (endpoints.stream().distinct().count() != endpoints.size()) { + throw new IllegalArgumentException("Expected all endpoints to be distinct, got " + endpoints); + } + this.endpoints = List.copyOf(endpoints); + } + + public List<Endpoint> asList() { + return endpoints; + } + + /** Returns the main endpoint, if any */ + public Optional<Endpoint> main() { + return endpoints.stream().filter(Predicate.not(Endpoint::legacy)).findFirst(); + } + + /** Returns the subset of endpoints are either legacy or not */ + public EndpointList legacy(boolean legacy) { + return of(endpoints.stream().filter(endpoint -> endpoint.legacy() == legacy)); + } + + /** Returns the subset of endpoints with given scope */ + public EndpointList scope(Endpoint.Scope scope) { + return of(endpoints.stream().filter(endpoint -> endpoint.scope() == scope)); + } + + /** Returns the union of this and given endpoints */ + public EndpointList and(EndpointList endpoints) { + return of(Stream.concat(asList().stream(), endpoints.asList().stream())); + } + + public static EndpointList of(Stream<Endpoint> endpoints) { + return new EndpointList(endpoints.collect(Collectors.toUnmodifiableList())); + } + + /** Returns the default global endpoints in given system. Default endpoints are served by a pre-provisioned routing layer */ + public static EndpointList defaultGlobal(ApplicationId application, SystemName system) { + // Rotation name is always default in the routing layer + RotationName rotation = RotationName.from("default"); + switch (system) { + case cd: + case main: + return new EndpointList(List.of( + Endpoint.of(application).target(rotation).on(Port.plain(4080)).legacy().in(system), + Endpoint.of(application).target(rotation).on(Port.tls(4443)).legacy().in(system), + Endpoint.of(application).target(rotation).on(Port.tls(4443)).in(system) + )); + } + return EMPTY; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java deleted file mode 100644 index ae638beed5c..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.RotationName; -import com.yahoo.config.provision.SystemName; - -import java.net.URI; -import java.util.Objects; - -/** - * Represents names for an application's global rotation. - * - * @author mpolden - */ -public class GlobalDnsName { - - // TODO: TLS: Remove all non-secure stuff when all traffic is on HTTPS. - public static final String DNS_SUFFIX = "global.vespa.yahooapis.com"; - public static final String OATH_DNS_SUFFIX = "global.vespa.oath.cloud"; - private static final int port = 4080; - private static final int securePort = 4443; - - private final URI url; - private final URI secureUrl; - private final URI oathUrl; - - public GlobalDnsName(ApplicationId application, SystemName system) { - this(application, system, null); - } - - public GlobalDnsName(ApplicationId application, SystemName system, RotationName rotation) { - Objects.requireNonNull(application, "application must be non-null"); - Objects.requireNonNull(system, "system must be non-null"); - - this.url = URI.create(String.format("http://%s%s%s.%s.%s:%d/", - clusterPart(rotation, "."), - systemPart(system, "."), - sanitize(application.application().value()), - sanitize(application.tenant().value()), - DNS_SUFFIX, - port)); - this.secureUrl = URI.create(String.format("https://%s%s%s--%s.%s:%d/", - clusterPart(rotation, "--"), - systemPart(system, "--"), - sanitize(application.application().value()), - sanitize(application.tenant().value()), - DNS_SUFFIX, - securePort)); - this.oathUrl = URI.create(String.format("https://%s%s%s--%s.%s:%d/", - clusterPart(rotation, "--"), - systemPart(system, "--"), - sanitize(application.application().value()), - sanitize(application.tenant().value()), - OATH_DNS_SUFFIX, - securePort)); - } - - /** URL to this rotation */ - public URI url() { - return url; - } - - /** HTTPS URL to this rotation */ - public URI secureUrl() { - return secureUrl; - } - - /** Oath HTTPS URL to this rotation */ - public URI oathUrl() { - return oathUrl; - } - - /** DNS name for this rotation */ - public String dnsName() { - return url.getHost(); - } - - /** DNS name for this rotation */ - public String secureDnsName() { - return secureUrl.getHost(); - } - - /** Oath DNS name for this rotation */ - public String oathDnsName() { - return oathUrl.getHost(); - } - - /** Sanitize by translating '_' to '-' as the former is not allowed in a DNS name */ - private static String sanitize(String s) { - return s.replace('_', '-'); - } - - private static String clusterPart(RotationName rotation, String separator) { - return rotation == null ? "" : rotation.value() + separator; - } - - private static String systemPart(SystemName system, String separator) { - return SystemName.main == system ? "" : system.name() + separator; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java index c459d519ab6..c4b69ce5588 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java @@ -6,14 +6,13 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; -import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; /** * Represents the DNS routing policy for a load balancer. @@ -41,9 +40,9 @@ public class RoutingPolicy { this.rotations = ImmutableSortedSet.copyOf(Objects.requireNonNull(rotations, "rotations must be non-null")); } - public RoutingPolicy(ApplicationId owner, ZoneId zone, ClusterSpec.Id cluster, HostName canonicalName, + public RoutingPolicy(ApplicationId owner, ZoneId zone, ClusterSpec.Id cluster, SystemName system, HostName canonicalName, Optional<String> dnsZone, Set<RotationName> rotations) { - this(owner, zone, HostName.from(aliasOf(cluster, owner, zone)), canonicalName, dnsZone, rotations); + this(owner, zone, HostName.from(endpointOf(cluster, owner, zone, system).dnsName()), canonicalName, dnsZone, rotations); } /** The application owning this */ @@ -76,6 +75,11 @@ public class RoutingPolicy { return rotations; } + /** Endpoints for this routing policy */ + public EndpointList endpointsIn(SystemName system) { + return EndpointList.of(rotations.stream().map(rotation -> endpointOf(owner, rotation, system))); + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -98,21 +102,14 @@ public class RoutingPolicy { zone.value()); } - /** Returns the alias to use for the given application cluster in zone */ - private static String aliasOf(ClusterSpec.Id cluster, ApplicationId application, ZoneId zone) { - List<String> parts = List.of(ignorePartIfDefault(cluster.value()), - ignorePartIfDefault(application.instance().value()), - application.application().value(), - application.tenant().value() + - "." + zone.value() + "." + "vespa.oath.cloud" - ); - return parts.stream() - .filter(Predicate.not(String::isBlank)) - .collect(Collectors.joining("--")); + /** Returns the endpoint of given rotation */ + public static Endpoint endpointOf(ApplicationId application, RotationName rotation, SystemName system) { + return Endpoint.of(application).target(rotation).on(Port.tls()).directRouting().in(system); } - private static String ignorePartIfDefault(String s) { - return "default".equalsIgnoreCase(s) ? "" : s; + /** Returns the endpoint of given cluster */ + public static Endpoint endpointOf(ClusterSpec.Id cluster, ApplicationId application, ZoneId zone, SystemName system) { + return Endpoint.of(application).target(cluster, zone).on(Port.tls()).directRouting().in(system); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java index 846c90a96f5..447f9a462b1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzClientFactoryImpl.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.athenz.impl; import com.google.inject.Inject; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zms.DefaultZmsClient; import com.yahoo.vespa.athenz.client.zms.ZmsClient; @@ -28,7 +29,7 @@ public class AthenzClientFactoryImpl implements AthenzClientFactory { } @Override - public AthenzService getControllerIdentity() { + public AthenzIdentity getControllerIdentity() { return identityProvider.identity(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 0732eeb97c3..75b7e137998 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.client.zms.RoleAction; import com.yahoo.vespa.athenz.client.zms.ZmsClient; @@ -19,9 +18,9 @@ import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.athenz.ApplicationAction; +import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; -import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; @@ -45,14 +44,14 @@ public class AthenzFacade implements AccessControl { private static final Logger log = Logger.getLogger(AthenzFacade.class.getName()); private final ZmsClient zmsClient; private final ZtsClient ztsClient; - private final AthenzService service; + private final AthenzIdentity service; @Inject public AthenzFacade(AthenzClientFactory factory) { this(factory.createZmsClient(), factory.createZtsClient(), factory.getControllerIdentity()); } - public AthenzFacade(ZmsClient zmsClient, ZtsClient ztsClient, AthenzService identity) { + public AthenzFacade(ZmsClient zmsClient, ZtsClient ztsClient, AthenzIdentity identity) { this.zmsClient = zmsClient; this.ztsClient = ztsClient; this.service = identity; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java index f7a8e702b06..37926d944b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZmsClientMock.java @@ -5,7 +5,6 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.client.zms.RoleAction; import com.yahoo.vespa.athenz.client.zms.ZmsClient; @@ -30,23 +29,23 @@ public class ZmsClientMock implements ZmsClient { private static final Logger log = Logger.getLogger(ZmsClientMock.class.getName()); private final AthenzDbMock athenz; - private final AthenzService controllerIdentity; + private final AthenzIdentity controllerIdentity; private static final Pattern TENANT_RESOURCE_PATTERN = Pattern.compile("service\\.hosting\\.tenant\\.(?<tenantDomain>[\\w\\-_]+)\\..*"); private static final Pattern APPLICATION_RESOURCE_PATTERN = Pattern.compile("service\\.hosting\\.tenant\\.[\\w\\-_]+\\.res_group\\.(?<resourceGroup>[\\w\\-_]+)\\.wildcard"); - public ZmsClientMock(AthenzDbMock athenz, AthenzService controllerIdentity) { + public ZmsClientMock(AthenzDbMock athenz, AthenzIdentity controllerIdentity) { this.athenz = athenz; this.controllerIdentity = controllerIdentity; } @Override - public void createTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + public void createTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token) { log("createTenancy(tenantDomain='%s')", tenantDomain); getDomainOrThrow(tenantDomain, false).isVespaTenant = true; } @Override - public void deleteTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + public void deleteTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token) { log("deleteTenancy(tenantDomain='%s')", tenantDomain); AthenzDbMock.Domain domain = getDomainOrThrow(tenantDomain, false); domain.isVespaTenant = false; @@ -55,7 +54,7 @@ public class ZmsClientMock implements ZmsClient { } @Override - public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { + public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { log("createProviderResourceGroup(tenantDomain='%s', resourceGroup='%s')", tenantDomain, resourceGroup); AthenzDbMock.Domain domain = getDomainOrThrow(tenantDomain, true); ApplicationId applicationId = new ApplicationId(resourceGroup); @@ -65,7 +64,7 @@ public class ZmsClientMock implements ZmsClient { } @Override - public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, OktaAccessToken token) { + public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, OktaAccessToken token) { log("deleteProviderResourceGroup(tenantDomain='%s', resourceGroup='%s')", tenantDomain, resourceGroup); getDomainOrThrow(tenantDomain, true).applications.remove(new ApplicationId(resourceGroup)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java index 8bb5ad12468..5c0407d35a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/mock/ZtsClientMock.java @@ -45,22 +45,22 @@ public class ZtsClientMock implements ZtsClient { } @Override - public InstanceIdentity registerInstance(AthenzService providerIdentity, AthenzService instanceIdentity, String instanceId, String attestationData, boolean requestServiceToken, Pkcs10Csr csr) { + public InstanceIdentity registerInstance(AthenzIdentity providerIdentity, AthenzIdentity instanceIdentity, String instanceId, String attestationData, boolean requestServiceToken, Pkcs10Csr csr) { throw new UnsupportedOperationException(); } @Override - public InstanceIdentity refreshInstance(AthenzService providerIdentity, AthenzService instanceIdentity, String instanceId, boolean requestServiceToken, Pkcs10Csr csr) { + public InstanceIdentity refreshInstance(AthenzIdentity providerIdentity, AthenzIdentity instanceIdentity, String instanceId, boolean requestServiceToken, Pkcs10Csr csr) { throw new UnsupportedOperationException(); } @Override - public Identity getServiceIdentity(AthenzService identity, String keyId, Pkcs10Csr csr) { + public Identity getServiceIdentity(AthenzIdentity identity, String keyId, Pkcs10Csr csr) { throw new UnsupportedOperationException(); } @Override - public Identity getServiceIdentity(AthenzService identity, String keyId, KeyPair keyPair, String dnsSuffix) { + public Identity getServiceIdentity(AthenzIdentity identity, String keyId, KeyPair keyPair, String dnsSuffix) { throw new UnsupportedOperationException(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 23e303a1b62..12290adbc12 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -12,6 +12,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.SystemName; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; @@ -21,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.api.ActivateResult; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; @@ -45,6 +48,7 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -115,6 +119,7 @@ public class InternalStepRunner implements StepRunner { case installTester: return installTester(id, logger); case startTests: return startTests(id, logger); case endTests: return endTests(id, logger); + case copyVespaLogs: return copyVespaLogs(id, logger); case deactivateReal: return deactivateReal(id, logger); case deactivateTester: return deactivateTester(id, logger); case report: return report(id, logger); @@ -396,7 +401,6 @@ public class InternalStepRunner implements StepRunner { controller.jobController().updateTestLog(id); - RunStatus status; TesterCloud.Status testStatus = controller.jobController().cloud().getStatus(testerEndpoint.get()); switch (testStatus) { case NOT_STARTED: @@ -405,17 +409,42 @@ public class InternalStepRunner implements StepRunner { return Optional.empty(); case FAILURE: logger.log("Tests failed."); - status = testFailure; break; + return Optional.of(testFailure); case ERROR: logger.log(INFO, "Tester failed running its tests!"); - status = error; break; + return Optional.of(error); case SUCCESS: logger.log("Tests completed successfully."); - status = running; break; + return Optional.of(running); default: throw new IllegalStateException("Unknown status '" + testStatus + "'!"); } - return Optional.of(status); + } + + private Optional<RunStatus> copyVespaLogs(RunId id, DualLogger logger) { + ZoneId zone = id.type().zone(controller.system()); + logger.log("Copying Vespa log from nodes of " + id.application() + " in " + zone + " ..."); + try { + List<LogEntry> entries = new ArrayList<>(); + String logs = IOUtils.readAll(controller.configServer().getLogStream(new DeploymentId(id.application(), zone), + Collections.emptyMap()), // Get all logs. + StandardCharsets.UTF_8); + for (String line : logs.split("\n")) { + String[] parts = line.split("\t"); + if (parts.length != 7) continue; + entries.add(new LogEntry(0, + (long) (Double.parseDouble(parts[0]) * 1000), + LogEntry.typeOf(LogLevel.parse(parts[5])), + parts[1] + '\t' + parts[3] + '\t' + parts[4] + '\n' + + parts[6].replaceAll("\\\\n", "\n") + .replaceAll("\\\\t", "\t"))); + } + controller.jobController().log(id, Step.copyVespaLogs, entries); + } + catch (Exception e) { + logger.log(INFO, "Failure getting vespa logs for " + id, e); + } + return Optional.of(running); // Don't let failure here stop cleanup. } private Optional<RunStatus> deactivateReal(RunId id, DualLogger logger) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index c8ba3f31316..5bff581c4ce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -109,18 +109,22 @@ public class JobController { } } - /** Stores the given log records for the given run and step. */ - public void log(RunId id, Step step, Level level, List<String> messages) { + /** Stores the given log entries for the given run and step. */ + public void log(RunId id, Step step, List<LogEntry> entries) { locked(id, __ -> { - List<LogEntry> entries = messages.stream() - .map(message -> new LogEntry(0, controller.clock().millis(), LogEntry.typeOf(level), message)) - .collect(toList()); logs.append(id.application(), id.type(), step, entries); return __; }); } - /** Stores the given log record for the given run and step. */ + /** Stores the given log messages for the given run and step. */ + public void log(RunId id, Step step, Level level, List<String> messages) { + log(id, step, messages.stream() + .map(message -> new LogEntry(0, controller.clock().millis(), LogEntry.typeOf(level), message)) + .collect(toList())); + } + + /** Stores the given log message for the given run and step. */ public void log(RunId id, Step step, Level level, String message) { log(id, step, level, Collections.singletonList(message)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java index 81f4934caaa..181ac2cdf96 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobProfile.java @@ -22,7 +22,8 @@ public enum JobProfile { installTester, startTests, endTests), - EnumSet.of(deactivateTester, + EnumSet.of(copyVespaLogs, + deactivateTester, deactivateReal, report)), @@ -34,7 +35,8 @@ public enum JobProfile { installTester, startTests, endTests), - EnumSet.of(deactivateTester, + EnumSet.of(copyVespaLogs, + deactivateTester, deactivateReal, report)), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 6ad73b25fe5..051a99074d1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -49,8 +49,11 @@ public enum Step { /** See that the tests are done running. */ endTests(startTests), + /** Fetch and store Vespa logs from the log server cluster of the deployment -- used for test deployments. */ + copyVespaLogs(deployInitialReal, deployReal, endTests), + /** Delete the real application -- used for test deployments. */ - deactivateReal(deployInitialReal, deployReal, endTests), + deactivateReal(deployInitialReal, deployReal, endTests, copyVespaLogs), /** Deactivate the tester. */ deactivateTester(deployTester, endTests), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 787a050e59e..0cf89d798a7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -65,15 +65,17 @@ public class DeploymentMetricsMaintainer extends Maintainer { .getDeploymentMetrics(application.id(), deployment.zone()); Instant now = controller().clock().instant(); applications.lockIfPresent(application.id(), locked -> { - DeploymentMetrics newMetrics = locked.get().deployments().get(deployment.zone()).metrics() - .withQueriesPerSecond(collectedMetrics.queriesPerSecond()) - .withWritesPerSecond(collectedMetrics.writesPerSecond()) - .withDocumentCount(collectedMetrics.documentCount()) - .withQueryLatencyMillis(collectedMetrics.queryLatencyMillis()) - .withWriteLatencyMillis(collectedMetrics.writeLatencyMillis()) - .at(now); - applications.store(locked.with(deployment.zone(), newMetrics) - .recordActivityAt(now, deployment.zone())); + Deployment existingDeployment = locked.get().deployments().get(deployment.zone()); + if (existingDeployment == null) return; // Deployment removed since we started collecting metrics + DeploymentMetrics newMetrics = existingDeployment.metrics() + .withQueriesPerSecond(collectedMetrics.queriesPerSecond()) + .withWritesPerSecond(collectedMetrics.writesPerSecond()) + .withDocumentCount(collectedMetrics.documentCount()) + .withQueryLatencyMillis(collectedMetrics.queryLatencyMillis()) + .withWriteLatencyMillis(collectedMetrics.writeLatencyMillis()) + .at(now); + applications.store(locked.with(existingDeployment.zone(), newMetrics) + .recordActivityAt(now, existingDeployment.zone())); }); } } catch (Exception e) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java index 7693f224b56..c2c68591dea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java @@ -5,7 +5,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.rotation.Rotation; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; @@ -84,8 +84,8 @@ public class DnsMaintainer extends Maintainer { /** Returns whether we can update the given record */ private static boolean canUpdate(Record record) { String recordName = record.name().asString(); - return recordName.endsWith(GlobalDnsName.DNS_SUFFIX) || - recordName.endsWith(GlobalDnsName.OATH_DNS_SUFFIX); + return recordName.endsWith(Endpoint.YAHOO_DNS_SUFFIX) || + recordName.endsWith(Endpoint.OATH_DNS_SUFFIX); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java index 9e10c5f9194..417a1944ad3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java @@ -15,7 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.RoutingId; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -91,7 +91,8 @@ public class RoutingPolicyMaintainer extends Maintainer { // Create DNS record for each routing ID for (Map.Entry<RoutingId, List<RoutingPolicy>> route : routingTable.entrySet()) { - GlobalDnsName dnsName = dnsName(route.getKey()); + Endpoint endpoint = RoutingPolicy.endpointOf(route.getKey().application(), route.getKey().rotation(), + controller().system()); Set<AliasTarget> targets = route.getValue() .stream() .filter(policy -> policy.dnsZone().isPresent()) @@ -100,10 +101,10 @@ public class RoutingPolicyMaintainer extends Maintainer { policy.zone())) .collect(Collectors.toSet()); try { - nameService.createAlias(RecordName.from(dnsName.oathDnsName()), targets); + nameService.createAlias(RecordName.from(endpoint.dnsName()), targets); } catch (Exception e) { log.log(LogLevel.WARNING, "Failed to create or update DNS record for global rotation " + - dnsName.oathDnsName() + ". Retrying in " + maintenanceInterval(), e); + endpoint.dnsName() + ". Retrying in " + maintenanceInterval(), e); } } } @@ -136,7 +137,8 @@ public class RoutingPolicyMaintainer extends Maintainer { /** Register DNS alias for given load balancer */ private RoutingPolicy registerCname(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer) { - RoutingPolicy routingPolicy = new RoutingPolicy(application, zone, loadBalancer.cluster(), + RoutingPolicy routingPolicy = new RoutingPolicy(application, zone, + loadBalancer.cluster(), controller().system(), loadBalancer.hostname(), loadBalancer.dnsZone(), loadBalancer.rotations()); RecordName name = RecordName.from(routingPolicy.alias().value()); @@ -186,23 +188,18 @@ public class RoutingPolicyMaintainer extends Maintainer { Set<RoutingId> activeRoutingIds = routingIdsFrom(loadBalancers); removalCandidates.removeAll(activeRoutingIds); for (RoutingId id : removalCandidates) { - GlobalDnsName dnsName = dnsName(id); + Endpoint endpoint = RoutingPolicy.endpointOf(id.application(), id.rotation(), controller().system()); try { - List<Record> records = nameService.findRecords(Record.Type.ALIAS, RecordName.from(dnsName.oathDnsName())); + List<Record> records = nameService.findRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName())); nameService.removeRecords(records); } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to remove all ALIAS records with name '" + dnsName.oathDnsName() + + log.log(LogLevel.WARNING, "Failed to remove all ALIAS records with name '" + endpoint.dnsName() + "'. Retrying in " + maintenanceInterval()); } } } } - /** Create a global DNS name for given routing ID */ - private GlobalDnsName dnsName(RoutingId routingId) { - return new GlobalDnsName(routingId.application(), controller().system(), routingId.rotation()); - } - /** Compute routing IDs from given load balancers */ private static Set<RoutingId> routingIdsFrom(Map<DeploymentId, List<LoadBalancer>> loadBalancers) { Set<RoutingId> routingIds = new LinkedHashSet<>(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 32c92e6f135..be464f95385 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -35,6 +35,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; @@ -196,6 +197,7 @@ class RunSerializer { case deployTester : return "deployTester"; case installTester : return "installTester"; case deactivateTester : return "deactivateTester"; + case copyVespaLogs : return "copyVespaLogs"; case startTests : return "startTests"; case endTests : return "endTests"; case report : return "report"; @@ -214,6 +216,7 @@ class RunSerializer { case "deployTester" : return deployTester; case "installTester" : return installTester; case "deactivateTester" : return deactivateTester; + case "copyVespaLogs" : return copyVespaLogs; case "startTests" : return startTests; case "endTests" : return endTests; case "report" : return report; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index c223d051237..01d9a01a316 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -268,7 +268,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { AthenzIdentityVerifier hostnameVerifier = new AthenzIdentityVerifier( singleton( - zoneRegistry.getConfigServerAthenzService( + zoneRegistry.getConfigServerAthenzIdentity( ZoneId.from(proxyRequest.getEnvironment(), proxyRequest.getRegion())))); return HttpClientBuilder.create() .setUserAgent("config-server-proxy-client") diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 44794600551..612c323fc31 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.RotationName; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -44,6 +43,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Logs; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -58,7 +58,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentCost; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; @@ -132,12 +132,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler { @Override public HttpResponse handle(HttpRequest request) { try { + Path path = new Path(request.getUri(), OPTIONAL_PREFIX); switch (request.getMethod()) { - case GET: return handleGET(request); - case PUT: return handlePUT(request); - case POST: return handlePOST(request); - case PATCH: return handlePATCH(request); - case DELETE: return handleDELETE(request); + case GET: return handleGET(path, request); + case PUT: return handlePUT(path, request); + case POST: return handlePOST(path, request); + case PATCH: return handlePATCH(path, request); + case DELETE: return handleDELETE(path, request); case OPTIONS: return handleOPTIONS(); default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); } @@ -163,8 +164,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } } - private HttpResponse handleGET(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + private HttpResponse handleGET(Path path, HttpRequest request) { if (path.matches("/application/v4/")) return root(request); if (path.matches("/application/v4/user")) return authenticatedUser(request); if (path.matches("/application/v4/tenant")) return tenants(request); @@ -173,6 +173,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); @@ -186,8 +187,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return ErrorResponse.notFoundError("Nothing at " + path); } - private HttpResponse handlePUT(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + private HttpResponse handlePUT(Path path, HttpRequest request) { if (path.matches("/application/v4/user")) return createUser(request); if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) @@ -195,8 +195,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return ErrorResponse.notFoundError("Nothing at " + path); } - private HttpResponse handlePOST(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + private HttpResponse handlePOST(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application"), request); @@ -214,15 +213,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return ErrorResponse.notFoundError("Nothing at " + path); } - private HttpResponse handlePATCH(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + private HttpResponse handlePATCH(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return setMajorVersion(path.get("tenant"), path.get("application"), request); return ErrorResponse.notFoundError("Nothing at " + path); } - private HttpResponse handleDELETE(HttpRequest request) { - Path path = new Path(request.getUri(), OPTIONAL_PREFIX); + private HttpResponse handleDELETE(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "all"); @@ -328,6 +325,58 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .orElseThrow(() -> new NotExistsException(applicationId + " not found")); } + private HttpResponse nodes(String tenantName, String applicationName, String instanceName, String environment, String region) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); + ZoneId zone = ZoneId.from(environment, region); + List<Node> nodes = controller.configServer().nodeRepository().list(zone, id); + + Slime slime = new Slime(); + Cursor nodesArray = slime.setObject().setArray("nodes"); + for (Node node : nodes) { + Cursor nodeObject = nodesArray.addObject(); + nodeObject.setString("hostname", node.hostname().value()); + nodeObject.setString("state", valueOf(node.state())); + nodeObject.setString("orchestration", valueOf(node.serviceState())); + nodeObject.setString("version", node.currentVersion().toString()); + nodeObject.setString("flavor", node.canonicalFlavor()); + nodeObject.setString("clusterId", node.clusterId()); + nodeObject.setString("clusterType", valueOf(node.clusterType())); + } + return new SlimeJsonResponse(slime); + } + + private static String valueOf(Node.State state) { + switch (state) { + case failed: return "failed"; + case parked: return "parked"; + case dirty: return "dirty"; + case ready: return "ready"; + case active: return "active"; + case inactive: return "inactive"; + case reserved: return "reserved"; + case provisioned: return "provisioned"; + default: throw new IllegalArgumentException("Unexpected node state '" + state + "'."); + } + } + + private static String valueOf(Node.ServiceState state) { + switch (state) { + case expectedUp: return "expectedUp"; + case allowedDown: return "allowedDown"; + case unorchestrated: return "unorchestrated"; + default: throw new IllegalArgumentException("Unexpected node state '" + state + "'."); + } + } + + private static String valueOf(Node.ClusterType type) { + switch (type) { + case admin: return "admin"; + case content: return "content"; + case container: return "container"; + default: throw new IllegalArgumentException("Unexpected node cluster type '" + type + "'."); + } + } + private HttpResponse logs(String tenantName, String applicationName, String instanceName, String environment, String region, Map<String, String> queryParameters) { ApplicationId application = ApplicationId.from(tenantName, applicationName, instanceName); ZoneId zone = ZoneId.from(environment, region); @@ -432,21 +481,23 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Rotation Cursor globalRotationsArray = object.setArray("globalRotations"); + application.endpointsIn(controller.system()) + .scope(Endpoint.Scope.global) + .legacy(false) // Hide legacy names + .asList().stream() + .map(Endpoint::url) + .map(URI::toString) + .forEach(globalRotationsArray::addString); - application.globalDnsName(controller.system()).ifPresent(rotation -> { - globalRotationsArray.addString(rotation.url().toString()); - globalRotationsArray.addString(rotation.secureUrl().toString()); - globalRotationsArray.addString(rotation.oathUrl().toString()); - object.setString("rotationId", application.rotation().get().asString()); - }); + application.rotation().ifPresent(rotation -> object.setString("rotationId", rotation.asString())); // Per-cluster rotations Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies(application.id()); for (RoutingPolicy policy : routingPolicies) { - for (RotationName rotation : policy.rotations()) { - GlobalDnsName dnsName = new GlobalDnsName(application.id(), controller.system(), rotation); - globalRotationsArray.addString(dnsName.oathUrl().toString()); - } + policy.endpointsIn(controller.system()).asList().stream() + .map(Endpoint::url) + .map(URI::toString) + .forEach(globalRotationsArray::addString); } // Deployments sorted according to deployment spec @@ -524,6 +575,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { response.setString("environment", deploymentId.zoneId().environment().value()); response.setString("region", deploymentId.zoneId().region().value()); + // serviceUrls contains zone/cluster-specific endpoints for this deployment. The name of these endpoints may + // contain the cluster name (if non-default) and since the controller has no knowledge of clusters, we have to + // ask the routing layer here Cursor serviceUrlArray = response.setArray("serviceUrls"); controller.applications().getDeploymentEndpoints(deploymentId) .ifPresent(endpoints -> endpoints.forEach(endpoint -> serviceUrlArray.addString(endpoint.toString()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 03ffdbb0208..f5532a964fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -42,6 +42,7 @@ import java.util.logging.Logger; public class UserApiHandler extends LoggingRequestHandler { private final static Logger log = Logger.getLogger(UserApiHandler.class.getName()); + private static final String optionalPrefix = "/api"; private final UserRoles roles; private final UserManagement users; @@ -56,10 +57,11 @@ public class UserApiHandler extends LoggingRequestHandler { @Override public HttpResponse handle(HttpRequest request) { try { + Path path = new Path(request.getUri(), optionalPrefix); switch (request.getMethod()) { - case GET: return handleGET(request); - case POST: return handlePOST(request); - case DELETE: return handleDELETE(request); + case GET: return handleGET(path, request); + case POST: return handlePOST(path, request); + case DELETE: return handleDELETE(path, request); case OPTIONS: return handleOPTIONS(); default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); } @@ -73,8 +75,7 @@ public class UserApiHandler extends LoggingRequestHandler { } } - private HttpResponse handleGET(HttpRequest request) { - Path path = new Path(request.getUri()); + private HttpResponse handleGET(Path path, HttpRequest request) { if (path.matches("/user/v1/tenant/{tenant}")) return listTenantRoleMembers(path.get("tenant")); if (path.matches("/user/v1/tenant/{tenant}/application/{application}")) return listApplicationRoleMembers(path.get("tenant"), path.get("application")); @@ -82,8 +83,7 @@ public class UserApiHandler extends LoggingRequestHandler { request.getUri().getPath())); } - private HttpResponse handlePOST(HttpRequest request) { - Path path = new Path(request.getUri()); + private HttpResponse handlePOST(Path path, HttpRequest request) { if (path.matches("/user/v1/tenant/{tenant}")) return addTenantRoleMember(path.get("tenant"), request); if (path.matches("/user/v1/tenant/{tenant}/application/{application}")) return addApplicationRoleMember(path.get("tenant"), path.get("application"), request); @@ -91,8 +91,7 @@ public class UserApiHandler extends LoggingRequestHandler { request.getUri().getPath())); } - private HttpResponse handleDELETE(HttpRequest request) { - Path path = new Path(request.getUri()); + private HttpResponse handleDELETE(Path path, HttpRequest request) { if (path.matches("/user/v1/tenant/{tenant}")) return removeTenantRoleMember(path.get("tenant"), request); if (path.matches("/user/v1/tenant/{tenant}/application/{application}")) return removeApplicationRoleMember(path.get("tenant"), path.get("application"), request); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java new file mode 100644 index 00000000000..16b875c1892 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -0,0 +1,114 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; +import org.junit.Test; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class EndpointTest { + + private static final ApplicationId app1 = ApplicationId.from("t1", "a1", "default"); + private static final ApplicationId app2 = ApplicationId.from("t2", "a2", "i2"); + + @Test + public void test_global_endpoints() { + RotationName rotation = RotationName.from("default"); // Always default for non-direct routing + + Map<String, Endpoint> tests = Map.of( + // Legacy endpoint + "http://a1.t1.global.vespa.yahooapis.com:4080/", + Endpoint.of(app1).target(rotation).on(Port.plain(4080)).legacy().in(SystemName.main), + + // Legacy endpoint with TLS + "https://a1--t1.global.vespa.yahooapis.com:4443/", + Endpoint.of(app1).target(rotation).on(Port.tls(4443)).legacy().in(SystemName.main), + + // Main endpoint + "https://a1--t1.global.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(rotation).on(Port.tls(4443)).in(SystemName.main), + + // Main endpoint in CD + "https://cd--a1--t1.global.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(rotation).on(Port.tls(4443)).in(SystemName.cd), + + // Main endpoint with direct routing and default TLS port + "https://a1.t1.global.vespa.oath.cloud/", + Endpoint.of(app1).target(rotation).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint with custom rotation name + "https://r1.a1.t1.global.vespa.oath.cloud/", + Endpoint.of(app1).target(RotationName.from("r1")).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint for custom instance in default rotation + "https://a2.t2.global.vespa.oath.cloud/", + Endpoint.of(app2).target(rotation).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint for custom instance with custom rotation name + "https://r2.a2.t2.global.vespa.oath.cloud/", + Endpoint.of(app2).target(RotationName.from("r2")).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint in public system + "https://a1.t1.global.public.vespa.oath.cloud/", + Endpoint.of(app1).target(rotation).on(Port.tls()).directRouting().in(SystemName.Public) + ); + tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); + } + + @Test + public void test_zone_endpoints() { + ClusterSpec.Id cluster = ClusterSpec.Id.from("default"); // Always default for non-direct routing + ZoneId prodZone = ZoneId.from("prod", "us-north-1"); + ZoneId testZone = ZoneId.from("test", "us-north-2"); + + Map<String, Endpoint> tests = Map.of( + // Legacy endpoint (always contains environment) + "http://a1.t1.us-north-1.prod.vespa.yahooapis.com:4080/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.plain(4080)).legacy().in(SystemName.main), + + // Secure legacy endpoint + "https://a1--t1.us-north-1.prod.vespa.yahooapis.com:4443/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.tls(4443)).legacy().in(SystemName.main), + + // Prod endpoint in main + "https://a1--t1.us-north-1.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.tls(4443)).in(SystemName.main), + + // Prod endpoint in CD + "https://cd--a1--t1.us-north-1.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(cluster, prodZone).on(Port.tls(4443)).in(SystemName.cd), + + // Test endpoint in main + "https://a1--t1.us-north-2.test.vespa.oath.cloud:4443/", + Endpoint.of(app1).target(cluster, testZone).on(Port.tls(4443)).in(SystemName.main), + + // Non-default cluster in main + "https://c1--a1--t1.us-north-1.vespa.oath.cloud/", + Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).in(SystemName.main), + + // Non-default instance in main + "https://i2--a2--t2.us-north-1.vespa.oath.cloud:4443/", + Endpoint.of(app2).target(cluster, prodZone).on(Port.tls(4443)).in(SystemName.main), + + // Non-default cluster in public + "https://c1.a1.t1.us-north-1.public.vespa.oath.cloud/", + Endpoint.of(app1).target(ClusterSpec.Id.from("c1"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public), + + // Non-default cluster and instance in public + "https://c2.i2.a2.t2.us-north-1.public.vespa.oath.cloud/", + Endpoint.of(app2).target(ClusterSpec.Id.from("c2"), prodZone).on(Port.tls()).directRouting().in(SystemName.Public) + ); + tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicyTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicyTest.java deleted file mode 100644 index 56c2f9bd968..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicyTest.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.zone.ZoneId; -import org.junit.Test; - -import java.util.Optional; -import java.util.Set; - -import static org.junit.Assert.assertEquals; - -/** - * @author mpolden - */ -public class RoutingPolicyTest { - - @Test - public void test_endpoint_names() { - ZoneId zoneId = ZoneId.from("prod", "us-north-1"); - ApplicationId withInstance = ApplicationId.from("tenant", "application", "instance"); - testAlias("instance--application--tenant.prod.us-north-1.vespa.oath.cloud", "default", withInstance, zoneId); - testAlias("cluster--instance--application--tenant.prod.us-north-1.vespa.oath.cloud", "cluster", withInstance, zoneId); - - ApplicationId withDefaultInstance = ApplicationId.from("tenant", "application", "default"); - testAlias("application--tenant.prod.us-north-1.vespa.oath.cloud", "default", withDefaultInstance, zoneId); - testAlias("cluster--application--tenant.prod.us-north-1.vespa.oath.cloud", "cluster", withDefaultInstance, zoneId); - } - - private void testAlias(String expected, String clusterName, ApplicationId applicationId, ZoneId zoneId) { - assertEquals(expected, new RoutingPolicy(applicationId, zoneId, ClusterSpec.Id.from(clusterName), - HostName.from("lb-0"), Optional.empty(), Set.of()).alias().value()); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index f606d904c5e..41f57a80ffd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -36,6 +36,7 @@ import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.debug; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.error; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.info; +import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.warning; import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.appId; import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.testerId; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; @@ -286,10 +287,43 @@ public class InternalStepRunnerTest { mailer.inbox("b@a").get(0).subject()); } + @Test + public void vespaLogIsCopied() { + RunId id = tester.startSystemTestTests(); + tester.cloud().set(TesterCloud.Status.ERROR); + tester.configServer().setLogStream(vespaLog); + long lastId = tester.jobs().details(id).get().lastId().getAsLong(); + tester.runner().run(); + assertEquals(failed, tester.jobs().run(id).get().steps().get(Step.endTests)); + assertTestLogEntries(id, Step.copyVespaLogs, + new LogEntry(lastId + 2, tester.clock().millis(), debug, "Copying Vespa log from nodes of tenant.application in zone test.us-east-1 in default ..."), + new LogEntry(lastId + 3, 1554970337084L, info, + "17480180-v6-3.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tContainer.com.yahoo.container.jdisc.ConfiguredApplication\n" + + "Switching to the latest deployed set of configurations and components. Application switch number: 2"), + new LogEntry(lastId + 4, 1554970337935L, info, + "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), + new LogEntry(lastId + 5, 1554970337947L, info, + "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), + new LogEntry(lastId + 6, 1554970337947L, info, + "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstdout\n" + + "ERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)"), + new LogEntry(lastId + 7, 1554970337947L, warning, + "17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tstderr\n" + + "java.lang.NullPointerException\n\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\n\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)")); + } + private void assertTestLogEntries(RunId id, Step step, LogEntry... entries) { assertEquals(List.of(entries), tester.jobs().details(id).get().get(step)); } + private static final String vespaLog = "1554970337.084804\t17480180-v6-3.ostk.bm2.prod.ne1.yahoo.com\t5549/832\tcontainer\tContainer.com.yahoo.container.jdisc.ConfiguredApplication\tinfo\tSwitching to the latest deployed set of configurations and components. Application switch number: 2\n" + + "1554970337.935104\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + + "1554970337.947777\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + + "1554970337.947820\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + + "1554970337.947844\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"; + @Test public void generates_correct_services_xml_test() { assertFile("test_runner_services.xml-cd", new String(InternalStepRunner.servicesXml(SystemName.cd, Optional.of("d-2-12-75")))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 61c397d893a..1222ae30c8a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -63,6 +63,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private Version lastPrepareVersion = null; private RuntimeException prepareException = null; private ConfigChangeActions configChangeActions = null; + private String log = "INFO - All good"; @Inject public ConfigServerMock(ZoneRegistryMock zoneRegistry) { @@ -248,7 +249,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer List<Node> nodes = nodeRepository.list(deployment.zoneId(), deployment.applicationId()); for (Node node : nodes) { nodeRepository.putByHostname(deployment.zoneId(), new Node(node.hostname(), - node.state(), node.type(), + Node.State.active, + node.type(), node.owner(), node.currentVersion(), application.version().get())); @@ -360,7 +362,11 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public InputStream getLogStream(DeploymentId deployment, Map<String, String> queryParameters) { - return IOUtils.toInputStream("INFO - All good"); + return IOUtils.toInputStream(log); + } + + public void setLogStream(String log) { + this.log = log; } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index ac15ab046c3..49bc910ac33 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -89,7 +89,10 @@ public class NodeRepositoryMock implements NodeRepository { node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), - node.wantedRebootGeneration())) + node.wantedRebootGeneration(), + node.canonicalFlavor(), + node.clusterId(), + node.clusterType())) .forEach(node -> putByHostname(zone, node)); } @@ -131,7 +134,10 @@ public class NodeRepositoryMock implements NodeRepository { node.restartGeneration(), node.wantedRestartGeneration() + 1, node.rebootGeneration(), - node.wantedRebootGeneration())); + node.wantedRebootGeneration(), + node.canonicalFlavor(), + node.clusterId(), + node.clusterType())); } public void doRestart(DeploymentId deployment, Optional<HostName> hostname) { @@ -147,7 +153,10 @@ public class NodeRepositoryMock implements NodeRepository { node.restartGeneration() + 1, node.wantedRestartGeneration(), node.rebootGeneration(), - node.wantedRebootGeneration())); + node.wantedRebootGeneration(), + node.canonicalFlavor(), + node.clusterId(), + node.clusterType())); } public void requestReboot(DeploymentId deployment, Optional<HostName> hostname) { @@ -163,7 +172,10 @@ public class NodeRepositoryMock implements NodeRepository { node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), - node.wantedRebootGeneration() + 1)); + node.wantedRebootGeneration() + 1, + node.canonicalFlavor(), + node.clusterId(), + node.clusterType())); } public void doReboot(DeploymentId deployment, Optional<HostName> hostname) { @@ -179,7 +191,10 @@ public class NodeRepositoryMock implements NodeRepository { node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration() + 1, - node.wantedRebootGeneration())); + node.wantedRebootGeneration(), + node.canonicalFlavor(), + node.clusterId(), + node.clusterType())); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 393268b4750..4248a513950 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -101,7 +101,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return ZoneFilterMock.from(Collections.unmodifiableList(zones)); } - public AthenzService getConfigServerAthenzService(ZoneId zone) { + public AthenzService getConfigServerAthenzIdentity(ZoneId zone) { return new AthenzService("vespadomain", "provider-" + zone.environment().value() + "-" + zone.region().value()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java index cbf50b65d1a..256ace4ae09 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java @@ -11,7 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.GlobalDnsName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; @@ -117,7 +117,7 @@ public class DnsMaintainerTest { for (int i = 1; i <= staleTotal; i++) { Rotation r = rotation(i); tester.controllerTester().nameService().createCname(RecordName.from("stale-record-" + i + "." + - GlobalDnsName.OATH_DNS_SUFFIX), + Endpoint.OATH_DNS_SUFFIX), RecordData.from(r.name() + ".")); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 843a4cfedd6..280a25acc08 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -45,6 +45,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; @@ -149,6 +150,10 @@ public class JobRunnerTest { outcomes.put(endTests, testFailure); runner.maintain(); assertTrue(run.get().hasFailed()); + assertEquals(List.of(copyVespaLogs, deactivateTester), run.get().readySteps()); + + outcomes.put(copyVespaLogs, running); + runner.maintain(); assertEquals(List.of(deactivateReal, deactivateTester), run.get().readySteps()); // Abortion does nothing, as the run has already failed. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index b053bfa1bfe..7a008d1f478 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -168,7 +168,7 @@ public class OsUpgraderTest { node.hostname(), node.state(), node.type(), node.owner(), node.currentVersion(), node.wantedVersion(), node.wantedOsVersion(), node.wantedOsVersion(), node.serviceState(), node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), - node.wantedRebootGeneration())); + node.wantedRebootGeneration(), node.canonicalFlavor(), node.clusterId(), node.clusterType())); } assertCurrent(version, application, zone); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java index 47d507e6094..0541a0b05f5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java @@ -6,11 +6,12 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -56,15 +57,22 @@ public class RoutingPolicyMaintainerTest { public void maintains_global_routing_policies() { int clustersPerZone = 2; tester.deployCompletely(app1, applicationPackage); - Map<Integer, Set<RotationName>> rotations = Map.of(0, Set.of(RotationName.from("r0"))); + // Cluster is member of 2 global rotations + Map<Integer, Set<RotationName>> rotations = Map.of(0, Set.of(RotationName.from("r0"), RotationName.from("r1"))); provisionLoadBalancers(app1, clustersPerZone, rotations); - // Creates alias record for cluster0 + // Creates alias records for cluster0 maintainer.maintain(); - Supplier<List<Record>> records1 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0--app1--tenant1.global.vespa.oath.cloud")); + Supplier<List<Record>> records1 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0.app1.tenant1.global.vespa.oath.cloud")); + Supplier<List<Record>> records2 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r1.app1.tenant1.global.vespa.oath.cloud")); assertEquals(2, records1.get().size()); + assertEquals(records1.get().size(), records2.get().size()); assertEquals("lb-0--tenant1:app1:default--prod.us-central-1.", records1.get().get(0).data().asString()); assertEquals("lb-0--tenant1:app1:default--prod.us-west-1.", records1.get().get(1).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-central-1.", records2.get().get(0).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-west-1.", records2.get().get(1).data().asString()); + assertEquals(2, tester.controller().applications().routingPolicies(app1.id()).iterator().next() + .endpointsIn(SystemName.main).asList().size()); // Applications gains a new deployment ApplicationPackage updatedApplicationPackage = new ApplicationPackageBuilder() @@ -85,13 +93,13 @@ public class RoutingPolicyMaintainerTest { assertEquals("lb-0--tenant1:app1:default--prod.us-west-1.", records1.get().get(2).data().asString()); // Another application is deployed - Supplier<List<Record>> records2 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0--app2--tenant1.global.vespa.oath.cloud")); + Supplier<List<Record>> records3 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0.app2.tenant1.global.vespa.oath.cloud")); tester.deployCompletely(app2, applicationPackage); provisionLoadBalancers(app2, 1, Map.of(0, Set.of(RotationName.from("r0")))); maintainer.maintain(); - assertEquals(2, records2.get().size()); - assertEquals("lb-0--tenant1:app2:default--prod.us-central-1.", records2.get().get(0).data().asString()); - assertEquals("lb-0--tenant1:app2:default--prod.us-west-1.", records2.get().get(1).data().asString()); + assertEquals(2, records3.get().size()); + assertEquals("lb-0--tenant1:app2:default--prod.us-central-1.", records3.get().get(0).data().asString()); + assertEquals("lb-0--tenant1:app2:default--prod.us-west-1.", records3.get().get(1).data().asString()); // All rotations for app1 are removed provisionLoadBalancers(app1, clustersPerZone, Collections.emptyMap()); @@ -101,7 +109,7 @@ public class RoutingPolicyMaintainerTest { assertEquals(clustersPerZone * numberOfDeployments, policies.size()); assertTrue("Rotation membership is removed from all policies", policies.stream().allMatch(policy -> policy.rotations().isEmpty())); - assertEquals("Rotations for " + app2 + " are not removed", 2, records2.get().size()); + assertEquals("Rotations for " + app2 + " are not removed", 2, records3.get().size()); } @Test @@ -114,10 +122,10 @@ public class RoutingPolicyMaintainerTest { // Creates records and policies for all clusters in all zones maintainer.maintain(); Set<String> expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); assertEquals(4, policies(app1).size()); @@ -131,12 +139,12 @@ public class RoutingPolicyMaintainerTest { provisionLoadBalancers(app1, clustersPerZone + 1); maintainer.maintain(); expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c2--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c2--app1--tenant1.prod.us-central-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); assertEquals(6, policies(app1).size()); @@ -146,16 +154,16 @@ public class RoutingPolicyMaintainerTest { provisionLoadBalancers(app2, clustersPerZone); maintainer.maintain(); expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c2--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c2--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c0--app2--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app2--tenant1.prod.us-central-1.vespa.oath.cloud", - "c0--app2--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app2--tenant1.prod.us-west-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-central-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); assertEquals(4, policies(app2).size()); @@ -164,14 +172,14 @@ public class RoutingPolicyMaintainerTest { provisionLoadBalancers(app1, clustersPerZone); maintainer.maintain(); expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c0--app2--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app2--tenant1.prod.us-central-1.vespa.oath.cloud", - "c0--app2--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app2--tenant1.prod.us-west-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-central-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); @@ -183,10 +191,10 @@ public class RoutingPolicyMaintainerTest { }); maintainer.maintain(); expectedRecords = Set.of( - "c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-west-1.vespa.oath.cloud", - "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud", - "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java index 03b432588bd..9cdaf925545 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializerTest.java @@ -25,9 +25,11 @@ import java.util.Optional; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static com.yahoo.vespa.hosted.controller.deployment.Step.copyVespaLogs; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deactivateTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployInitialReal; @@ -98,6 +100,7 @@ public class RunSerializerTest { .put(deployTester, succeeded) .put(installTester, unfinished) .put(deactivateTester, failed) + .put(copyVespaLogs, succeeded) .put(startTests, succeeded) .put(endTests, unfinished) .put(report, failed) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json index cda3834d47d..880d2ebf527 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/run-status.json @@ -15,6 +15,7 @@ "deployTester": "succeeded", "installTester": "unfinished", "deactivateTester": "failed", + "copyVespaLogs": "succeeded", "startTests": "succeeded", "endTests": "unfinished", "report": "failed" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java index caa44b033af..95477758deb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java @@ -24,14 +24,20 @@ public class ControllerContainerCloudTest extends ControllerContainerTest { } @Override - protected String securityXml() { + protected String variablePartXml() { return " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControlRequests'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.security.CloudAccessControl'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockUserManagement'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMarketplace'/>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + + " <binding>http://*/application/v4/*</binding>\n" + + " <binding>http://*/api/application/v4/*</binding>\n" + + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.user.UserApiHandler'>\n" + " <binding>http://*/user/v1/*</binding>\n" + + " <binding>http://*/api/user/v1/*</binding>\n" + " </handler>\n" + " <http>\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 716cf622e76..6abfa7fa72d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -97,9 +97,6 @@ public class ControllerContainerTest { " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.api.role.Roles'/>\n" + - " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + - " <binding>http://*/application/v4/*</binding>\n" + - " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.deployment.DeploymentApiHandler'>\n" + " <binding>http://*/deployment/v1/*</binding>\n" + " </handler>\n" + @@ -123,7 +120,7 @@ public class ControllerContainerTest { " <binding>http://*/zone/v2</binding>\n" + " <binding>http://*/zone/v2/*</binding>\n" + " </handler>\n" + - securityXml() + + variablePartXml() + "</jdisc>"; } @@ -131,10 +128,13 @@ public class ControllerContainerTest { return SystemName.main; } - protected String securityXml() { + protected String variablePartXml() { return " <component id='com.yahoo.vespa.hosted.controller.security.AthenzAccessControlRequests'/>\n" + " <component id='com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade'/>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" + + " <binding>http://*/application/v4/*</binding>\n" + + " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.athenz.AthenzApiHandler'>\n" + " <binding>http://*/athenz/v1/*</binding>\n" + " </handler>\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index ec17e26d867..f617dc0a447 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -380,6 +380,11 @@ public class ApplicationApiTest extends ControllerContainerTest { .recursive("true"), new File("application1-recursive.json")); + // GET nodes + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/nodes", GET) + .userIdentity(USER_ID), + new File("application-nodes.json")); + // GET logs tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/environment/prod/region/us-central-1/instance/default/logs?from=1233&to=3214", GET) .userIdentity(USER_ID), @@ -1327,7 +1332,8 @@ public class ApplicationApiTest extends ControllerContainerTest { public void applicationWithPerClusterGlobalRotation() { Application app = controllerTester.createApplication(); RoutingPolicy policy = new RoutingPolicy(app.id(), ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - ClusterSpec.Id.from("default"), HostName.from("lb-0-canonical-name"), + ClusterSpec.Id.from("default"), controllerTester.controller().system(), + HostName.from("lb-0-canonical-name"), Optional.of("dns-zone-1"), Set.of(RotationName.from("c0"))); tester.controller().curator().writeRoutingPolicies(app.id(), Set.of(policy)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json index cd531bb96da..baaf0cd038d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-cluster-global-rotation.json @@ -8,7 +8,7 @@ "changeBlockers": [], "compileVersion": "(ignore)", "globalRotations": [ - "https://c0--application1--tenant1.global.vespa.oath.cloud:4443/" + "https://c0.application1.tenant1.global.vespa.oath.cloud/" ], "instances": [], "metrics": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json new file mode 100644 index 00000000000..eb53ff7161e --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json @@ -0,0 +1,13 @@ +{ + "nodes": [ + { + "hostname": "host-tenant1:application1:default-prod.us-central-1", + "state": "active", + "orchestration": "unorchestrated", + "version": "6.1", + "flavor": "d-2-8-50", + "clusterId": "cluster", + "clusterType": "container" + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json index f2f38f7f509..b52fec761d8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json @@ -231,8 +231,6 @@ "changeBlockers": [], "compileVersion": "(ignore)", "globalRotations": [ - "http://application1.tenant1.global.vespa.yahooapis.com:4080/", - "https://application1--tenant1.global.vespa.yahooapis.com:4443/", "https://application1--tenant1.global.vespa.oath.cloud:4443/" ], "rotationId": "rotation-id-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 22e8573b1d4..4b2cb397b5b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -219,8 +219,6 @@ ], "compileVersion": "(ignore)", "globalRotations": [ - "http://application1.tenant1.global.vespa.yahooapis.com:4080/", - "https://application1--tenant1.global.vespa.yahooapis.com:4443/", "https://application1--tenant1.global.vespa.oath.cloud:4443/" ], "rotationId": "rotation-id-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json index 662e045d169..fa903b61825 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json @@ -219,8 +219,6 @@ ], "compileVersion": "(ignore)", "globalRotations": [ - "http://application1.tenant1.global.vespa.yahooapis.com:4080/", - "https://application1--tenant1.global.vespa.yahooapis.com:4443/", "https://application1--tenant1.global.vespa.oath.cloud:4443/" ], "rotationId": "rotation-id-1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 1c8da68e138..5ac896e2753 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -57,6 +57,7 @@ "installTester": "unfinished", "startTests": "unfinished", "endTests": "unfinished", + "copyVespaLogs": "unfinished", "deactivateReal": "unfinished", "deactivateTester": "unfinished", "report": "unfinished" @@ -94,6 +95,7 @@ "installTester": "unfinished", "startTests": "unfinished", "endTests": "unfinished", + "copyVespaLogs": "unfinished", "deactivateReal": "unfinished", "deactivateTester": "unfinished", "report": "unfinished" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json index c7580d869b0..84866650afc 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json @@ -116,6 +116,7 @@ "installTester": "succeeded", "startTests": "succeeded", "endTests": "succeeded", + "copyVespaLogs": "succeeded", "deactivateReal": "succeeded", "deactivateTester": "succeeded", "report": "succeeded" @@ -159,6 +160,7 @@ "installTester": "succeeded", "startTests": "succeeded", "endTests": "succeeded", + "copyVespaLogs": "succeeded", "deactivateReal": "succeeded", "deactivateTester": "succeeded", "report": "succeeded" @@ -192,6 +194,7 @@ "installTester": "succeeded", "startTests": "succeeded", "endTests": "succeeded", + "copyVespaLogs": "succeeded", "deactivateReal": "succeeded", "deactivateTester": "succeeded", "report": "succeeded" @@ -269,6 +272,7 @@ "installTester": "unfinished", "startTests": "unfinished", "endTests": "unfinished", + "copyVespaLogs": "succeeded", "deactivateReal": "succeeded", "deactivateTester": "succeeded", "report": "succeeded" @@ -310,6 +314,7 @@ "installTester": "succeeded", "startTests": "succeeded", "endTests": "succeeded", + "copyVespaLogs": "succeeded", "deactivateReal": "succeeded", "deactivateTester": "succeeded", "report": "succeeded" @@ -355,6 +360,7 @@ "installTester": "succeeded", "startTests": "succeeded", "endTests": "succeeded", + "copyVespaLogs": "succeeded", "deactivateReal": "succeeded", "deactivateTester": "succeeded", "report": "succeeded" @@ -390,6 +396,7 @@ "installTester": "succeeded", "startTests": "succeeded", "endTests": "succeeded", + "copyVespaLogs": "succeeded", "deactivateReal": "succeeded", "deactivateTester": "succeeded", "report": "succeeded" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json index 8c5e5253482..68599618ab4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json @@ -23,6 +23,7 @@ "installTester": "succeeded", "startTests": "succeeded", "endTests": "succeeded", + "copyVespaLogs": "succeeded", "deactivateReal": "succeeded", "deactivateTester": "succeeded", "report": "succeeded" @@ -68,6 +69,7 @@ "installTester": "succeeded", "startTests": "succeeded", "endTests": "succeeded", + "copyVespaLogs": "succeeded", "deactivateReal": "succeeded", "deactivateTester": "succeeded", "report": "succeeded" @@ -113,6 +115,7 @@ "installTester": "succeeded", "startTests": "succeeded", "endTests": "succeeded", + "copyVespaLogs": "succeeded", "deactivateReal": "succeeded", "deactivateTester": "succeeded", "report": "succeeded" @@ -158,6 +161,7 @@ "installTester": "unfinished", "startTests": "unfinished", "endTests": "unfinished", + "copyVespaLogs": "succeeded", "deactivateReal": "succeeded", "deactivateTester": "succeeded", "report": "succeeded" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json index 71f647ead11..2f5c6e489bd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json @@ -20,6 +20,7 @@ "installTester": "unfinished", "startTests": "unfinished", "endTests": "unfinished", + "copyVespaLogs": "unfinished", "deactivateReal": "unfinished", "deactivateTester": "unfinished", "report": "unfinished" diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index 1eb3316a33e..bc8dd8d4479 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -149,7 +149,7 @@ public class OsApiTest extends ControllerContainerTest { node.hostname(), node.state(), node.type(), node.owner(), node.currentVersion(), node.wantedVersion(), node.wantedOsVersion(), node.wantedOsVersion(), node.serviceState(), node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), - node.wantedRebootGeneration())); + node.wantedRebootGeneration(), node.canonicalFlavor(), node.clusterId(), node.clusterType())); } } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index caf0ec82aae..3a78e9fc262 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -42,6 +42,11 @@ public class UserApiTest extends ControllerContainerCloudTest { .roles(operator), "[]"); + // GET at application/v4/tenant is available also under the /api prefix. + tester.assertResponse(request("/api/application/v4/tenant") + .roles(operator), + "[]"); + // POST a tenant is not available to everyone. tester.assertResponse(request("/application/v4/tenant/my-tenant", POST) .data("{\"token\":\"hello\"}"), @@ -120,6 +125,11 @@ public class UserApiTest extends ControllerContainerCloudTest { .roles(Set.of(roles.tenantOperator(id.tenant()))), new File("application-roles.json")); + // GET application role information is available also under the /api prefix. + tester.assertResponse(request("/api/user/v1/tenant/my-tenant/application/my-app") + .roles(Set.of(roles.tenantOperator(id.tenant()))), + new File("application-roles.json")); + // DELETE an application role is allowed for an application admin. tester.assertResponse(request("/user/v1/tenant/my-tenant/application/my-app", DELETE) .roles(Set.of(roles.applicationAdmin(id.tenant(), id.application()))) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java index 666c7774cf5..8d1f40260e3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java @@ -65,8 +65,8 @@ public class RotationRepositoryTest { application = tester.applications().require(application.id()); assertEquals(expected.id(), application.rotation().get()); - assertEquals(URI.create("http://app1.tenant1.global.vespa.yahooapis.com:4080/"), - application.globalDnsName(SystemName.main).get().url()); + assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), + application.endpointsIn(SystemName.main).main().get().url()); try (RotationLock lock = repository.lock()) { Rotation rotation = repository.getOrAssignRotation(tester.applications().require(application.id()), lock); assertEquals(expected, rotation); @@ -153,10 +153,9 @@ public class RotationRepositoryTest { Application application = tester.createApplication("app2", "tenant2", 22L, 2L); tester.deployCompletely(application, applicationPackage); - assertEquals(new RotationId("foo-1"), tester.applications().require(application.id()) - .rotation().get()); - assertEquals("https://cd--app2--tenant2.global.vespa.yahooapis.com:4443/", tester.applications().require(application.id()) - .globalDnsName(SystemName.cd).get().secureUrl().toString()); + assertEquals(new RotationId("foo-1"), tester.applications().require(application.id()).rotation().get()); + assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/", tester.applications().require(application.id()) + .endpointsIn(SystemName.cd).main().get().url().toString()); } } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java index 07efd419c97..ec74fb9246d 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingHandler.java @@ -31,7 +31,13 @@ import com.yahoo.processing.execution.chain.ChainRegistry; import com.yahoo.statistics.Statistics; import java.util.TimerTask; -import java.util.concurrent.*; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.PriorityBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.TimeUnit; import java.util.logging.Logger; import static com.yahoo.component.chain.ChainsConfigurer.prepareChainRegistry; @@ -203,8 +209,7 @@ public class DocumentProcessingHandler extends AbstractRequestHandler { return null; } - @SuppressWarnings("unchecked") - void submit(DocumentProcessingTask task) { + private void submit(DocumentProcessingTask task) { if (threadPool.isAboveLimit()) { task.queueFull(); } else { diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java index 0bc95fe6c7b..ca4648678a5 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/DocumentProcessingTask.java @@ -17,8 +17,6 @@ import java.io.StringWriter; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.Objects; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; @@ -34,7 +32,6 @@ public class DocumentProcessingTask implements Comparable<DocumentProcessingTask private final DocumentProcessingHandler docprocHandler; private RequestContext requestContext; - private int waitCounter; private final static AtomicLong seq = new AtomicLong(); private final long seqNum; @@ -45,7 +42,6 @@ public class DocumentProcessingTask implements Comparable<DocumentProcessingTask seqNum = seq.getAndIncrement(); this.requestContext = requestContext; this.docprocHandler = docprocHandler; - this.waitCounter = 10; this.service = service; } @@ -80,18 +76,6 @@ public class DocumentProcessingTask implements Comparable<DocumentProcessingTask } /** - * Used by DocprocThreadManager. If a ProcessingTask has been taken by a thread, it can wait() no longer than - * waitCounter (currently 10) times before being executed. This is to prevent large tasks from being delayed - * forever. - * - * @return true if this task can wait, false if it must run NOW. - */ - boolean doWait() { - --waitCounter; - return (waitCounter > 0); - } - - /** * Processes a single Processing, and fails the message if this processing fails. * * @param executor the DocprocService to use for processing diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java index edb426c6392..ff891dbb298 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/MessageBusDocumentAccess.java @@ -1,18 +1,22 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.documentapi.messagebus; -import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.document.select.parser.ParseException; -import com.yahoo.documentapi.*; +import com.yahoo.documentapi.AsyncParameters; +import com.yahoo.documentapi.DocumentAccess; +import com.yahoo.documentapi.DocumentAccessException; +import com.yahoo.documentapi.SubscriptionParameters; +import com.yahoo.documentapi.SubscriptionSession; +import com.yahoo.documentapi.SyncParameters; +import com.yahoo.documentapi.VisitorDestinationParameters; +import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.messagebus.MessageBus; import com.yahoo.messagebus.NetworkMessageBus; import com.yahoo.messagebus.RPCMessageBus; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.local.LocalNetwork; -import com.yahoo.messagebus.network.local.LocalWire; -import com.yahoo.messagebus.routing.RoutingTable; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java index 6bd5d658509..0c40f527287 100644 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/loadtypes/LoadTypeSet.java @@ -2,7 +2,6 @@ package com.yahoo.documentapi.messagebus.loadtypes; import com.yahoo.config.subscription.ConfigGetter; -import com.yahoo.config.subscription.ConfigSubscriber; import com.yahoo.vespa.config.content.LoadTypeConfig; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import java.security.MessageDigest; diff --git a/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp b/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp index c3b42124155..aa4c3b8c021 100644 --- a/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp +++ b/eval/src/tests/eval/simple_tensor/simple_tensor_test.cpp @@ -4,6 +4,8 @@ #include <vespa/eval/eval/simple_tensor_engine.h> #include <vespa/eval/eval/operation.h> #include <vespa/vespalib/util/stash.h> +#include <vespa/vespalib/data/memory.h> +#include <vespa/vespalib/objects/nbostream.h> #include <iostream> using namespace vespalib::eval; @@ -12,6 +14,8 @@ using Cell = SimpleTensor::Cell; using Cells = SimpleTensor::Cells; using Address = SimpleTensor::Address; using Stash = vespalib::Stash; +using vespalib::nbostream; +using vespalib::Memory; TensorSpec to_spec(const Value &a) { return SimpleTensorEngine::ref().to_spec(a); } @@ -143,4 +147,208 @@ TEST("require that simple tensors support dimension reduction") { EXPECT_NOT_EQUAL(to_spec(*result_sum_y), to_spec(*result_sum_x)); } +//----------------------------------------------------------------------------- + +struct SparseTensorExample { + TensorSpec make_spec() const { + return TensorSpec("tensor(x{},y{})") + .add({{"x","a"},{"y","a"}}, 1) + .add({{"x","a"},{"y","b"}}, 2) + .add({{"x","b"},{"y","a"}}, 3); + } + std::unique_ptr<SimpleTensor> make_tensor() const { + return SimpleTensor::create(make_spec()); + } + template <typename T> + void encode_inner(nbostream &dst) const { + dst.putInt1_4Bytes(2); + dst.writeSmallString("x"); + dst.writeSmallString("y"); + dst.putInt1_4Bytes(3); + dst.writeSmallString("a"); + dst.writeSmallString("a"); + dst << (T) 1; + dst.writeSmallString("a"); + dst.writeSmallString("b"); + dst << (T) 2; + dst.writeSmallString("b"); + dst.writeSmallString("a"); + dst << (T) 3; + } + void encode_default(nbostream &dst) const { + dst.putInt1_4Bytes(1); + encode_inner<double>(dst); + } + void encode_with_double(nbostream &dst) const { + dst.putInt1_4Bytes(5); + dst.putInt1_4Bytes(0); + encode_inner<double>(dst); + } + void encode_with_float(nbostream &dst) const { + dst.putInt1_4Bytes(5); + dst.putInt1_4Bytes(1); + encode_inner<float>(dst); + } +}; + +TEST_F("require that sparse tensors can be decoded", SparseTensorExample()) { + nbostream data1; + nbostream data2; + nbostream data3; + f1.encode_default(data1); + f1.encode_with_double(data2); + f1.encode_with_float(data3); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data1)), f1.make_spec()); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data2)), f1.make_spec()); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data3)), f1.make_spec()); +} + +TEST_F("require that sparse tensors can be encoded", SparseTensorExample()) { + nbostream data; + nbostream expect; + SimpleTensor::encode(*f1.make_tensor(), data); + f1.encode_default(expect); + EXPECT_EQUAL(Memory(data.peek(), data.size()), Memory(expect.peek(), expect.size())); +} + +//----------------------------------------------------------------------------- + +struct DenseTensorExample { + TensorSpec make_spec() const { + return TensorSpec("tensor(x[3],y[2])") + .add({{"x",0},{"y",0}}, 1) + .add({{"x",0},{"y",1}}, 2) + .add({{"x",1},{"y",0}}, 3) + .add({{"x",1},{"y",1}}, 4) + .add({{"x",2},{"y",0}}, 5) + .add({{"x",2},{"y",1}}, 6); + } + std::unique_ptr<SimpleTensor> make_tensor() const { + return SimpleTensor::create(make_spec()); + } + template <typename T> + void encode_inner(nbostream &dst) const { + dst.putInt1_4Bytes(2); + dst.writeSmallString("x"); + dst.putInt1_4Bytes(3); + dst.writeSmallString("y"); + dst.putInt1_4Bytes(2); + dst << (T) 1; + dst << (T) 2; + dst << (T) 3; + dst << (T) 4; + dst << (T) 5; + dst << (T) 6; + } + void encode_default(nbostream &dst) const { + dst.putInt1_4Bytes(2); + encode_inner<double>(dst); + } + void encode_with_double(nbostream &dst) const { + dst.putInt1_4Bytes(6); + dst.putInt1_4Bytes(0); + encode_inner<double>(dst); + } + void encode_with_float(nbostream &dst) const { + dst.putInt1_4Bytes(6); + dst.putInt1_4Bytes(1); + encode_inner<float>(dst); + } +}; + +TEST_F("require that dense tensors can be decoded", DenseTensorExample()) { + nbostream data1; + nbostream data2; + nbostream data3; + f1.encode_default(data1); + f1.encode_with_double(data2); + f1.encode_with_float(data3); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data1)), f1.make_spec()); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data2)), f1.make_spec()); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data3)), f1.make_spec()); +} + +TEST_F("require that dense tensors can be encoded", DenseTensorExample()) { + nbostream data; + nbostream expect; + SimpleTensor::encode(*f1.make_tensor(), data); + f1.encode_default(expect); + EXPECT_EQUAL(Memory(data.peek(), data.size()), Memory(expect.peek(), expect.size())); +} + +//----------------------------------------------------------------------------- + +struct MixedTensorExample { + TensorSpec make_spec() const { + return TensorSpec("tensor(x{},y{},z[2])") + .add({{"x","a"},{"y","a"},{"z",0}}, 1) + .add({{"x","a"},{"y","a"},{"z",1}}, 2) + .add({{"x","a"},{"y","b"},{"z",0}}, 3) + .add({{"x","a"},{"y","b"},{"z",1}}, 4) + .add({{"x","b"},{"y","a"},{"z",0}}, 5) + .add({{"x","b"},{"y","a"},{"z",1}}, 6); + } + std::unique_ptr<SimpleTensor> make_tensor() const { + return SimpleTensor::create(make_spec()); + } + template <typename T> + void encode_inner(nbostream &dst) const { + dst.putInt1_4Bytes(2); + dst.writeSmallString("x"); + dst.writeSmallString("y"); + dst.putInt1_4Bytes(1); + dst.writeSmallString("z"); + dst.putInt1_4Bytes(2); + dst.putInt1_4Bytes(3); + dst.writeSmallString("a"); + dst.writeSmallString("a"); + dst << (T) 1; + dst << (T) 2; + dst.writeSmallString("a"); + dst.writeSmallString("b"); + dst << (T) 3; + dst << (T) 4; + dst.writeSmallString("b"); + dst.writeSmallString("a"); + dst << (T) 5; + dst << (T) 6; + } + void encode_default(nbostream &dst) const { + dst.putInt1_4Bytes(3); + encode_inner<double>(dst); + } + void encode_with_double(nbostream &dst) const { + dst.putInt1_4Bytes(7); + dst.putInt1_4Bytes(0); + encode_inner<double>(dst); + } + void encode_with_float(nbostream &dst) const { + dst.putInt1_4Bytes(7); + dst.putInt1_4Bytes(1); + encode_inner<float>(dst); + } +}; + +TEST_F("require that mixed tensors can be decoded", MixedTensorExample()) { + nbostream data1; + nbostream data2; + nbostream data3; + f1.encode_default(data1); + f1.encode_with_double(data2); + f1.encode_with_float(data3); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data1)), f1.make_spec()); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data2)), f1.make_spec()); + EXPECT_EQUAL(to_spec(*SimpleTensor::decode(data3)), f1.make_spec()); +} + +TEST_F("require that mixed tensors can be encoded", MixedTensorExample()) { + nbostream data; + nbostream expect; + SimpleTensor::encode(*f1.make_tensor(), data); + f1.encode_default(expect); + EXPECT_EQUAL(Memory(data.peek(), data.size()), Memory(expect.peek(), expect.size())); +} + +//----------------------------------------------------------------------------- + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/eval/src/vespa/eval/eval/simple_tensor.cpp b/eval/src/vespa/eval/eval/simple_tensor.cpp index 1836f2088f3..1bce8983666 100644 --- a/eval/src/vespa/eval/eval/simple_tensor.cpp +++ b/eval/src/vespa/eval/eval/simple_tensor.cpp @@ -19,6 +19,9 @@ using CellRef = std::reference_wrapper<const Cell>; namespace { +constexpr uint32_t DOUBLE_CELL_TYPE = 0; +constexpr uint32_t FLOAT_CELL_TYPE = 1; + void assert_type(const ValueType &type) { (void) type; assert(!type.is_abstract()); @@ -418,14 +421,17 @@ public: struct Format { bool is_sparse; bool is_dense; + bool with_cell_type; uint32_t tag; explicit Format(const TypeMeta &meta) : is_sparse(meta.mapped.size() > 0), is_dense((meta.indexed.size() > 0) || !is_sparse), + with_cell_type(false), tag((is_sparse ? 0x1 : 0) | (is_dense ? 0x2 : 0)) {} explicit Format(uint32_t tag_in) : is_sparse((tag_in & 0x1) != 0), is_dense((tag_in & 0x2) != 0), + with_cell_type((tag_in & 0x4) != 0), tag(tag_in) {} ~Format() {} }; @@ -458,6 +464,13 @@ void encode_mapped_labels(nbostream &output, const TypeMeta &meta, const Address } } +uint32_t maybe_decode_cell_type(nbostream &input, const Format &format) { + if (format.with_cell_type) { + return input.getInt1_4Bytes(); + } + return DOUBLE_CELL_TYPE; +} + ValueType decode_type(nbostream &input, const Format &format) { std::vector<ValueType::Dimension> dim_list; if (format.is_sparse) { @@ -496,17 +509,20 @@ void decode_mapped_labels(nbostream &input, const TypeMeta &meta, Address &addr) } } -void decode_cells(nbostream &input, const ValueType &type, const TypeMeta meta, +void decode_cells(uint32_t cell_type, nbostream &input, const ValueType &type, const TypeMeta meta, Address &address, size_t n, Builder &builder) { if (n < meta.indexed.size()) { Label &label = address[meta.indexed[n]]; size_t size = type.dimensions()[meta.indexed[n]].size; for (label.index = 0; label.index < size; ++label.index) { - decode_cells(input, type, meta, address, n + 1, builder); + decode_cells(cell_type, input, type, meta, address, n + 1, builder); } } else { - builder.set(address, input.readValue<double>()); + double value = (cell_type == FLOAT_CELL_TYPE) + ? input.readValue<float>() + : input.readValue<double>(); + builder.set(address, value); } } @@ -693,6 +709,7 @@ std::unique_ptr<SimpleTensor> SimpleTensor::decode(nbostream &input) { Format format(input.getInt1_4Bytes()); + uint32_t cell_type = maybe_decode_cell_type(input, format); ValueType type = decode_type(input, format); TypeMeta meta(type); Builder builder(type); @@ -700,7 +717,7 @@ SimpleTensor::decode(nbostream &input) Address address(type.dimensions().size(), Label(size_t(0))); for (size_t i = 0; i < num_blocks; ++i) { decode_mapped_labels(input, meta, address); - decode_cells(input, type, meta, address, 0, builder); + decode_cells(cell_type, input, type, meta, address, 0, builder); } return builder.build(); } diff --git a/eval/src/vespa/eval/tensor/serialization/format.txt b/eval/src/vespa/eval/tensor/serialization/format.txt index 780f88af01a..1a454b0ccf8 100644 --- a/eval/src/vespa/eval/tensor/serialization/format.txt +++ b/eval/src/vespa/eval/tensor/serialization/format.txt @@ -4,15 +4,25 @@ interpreted as a single unified binary format. The description below uses data types defined by document serialization (nbostream) combined with some comments and python-inspired flow-control. The mixed[3] binary format is defined in such a way that it overlays as -effortlessly as possible with both existing formats. +effortlessly as possible with the sparse[1] and dense[2] formats. + +All format archetypes can also be encoded with values other than +double. Using a separate bit to specify that the format includes cells +with a specific type gives rise to 3 new formats: +sparse_with_cell_type[5], dense_with_cell_type[6] and +mixed_with_cell_type[7]. //----------------------------------------------------------------------------- -1_4_int: type (1:sparse, 2:dense, 3:mixed) +1_4_int: type (1/5:sparse, 2/6:dense, 3/7:mixed) bit 0 -> 'sparse' bit 1 -> 'dense' + bit 2 -> 'with_cell_type' (mixed tensors are tagged as both 'sparse' and 'dense') +if ('with_cell_type') + 1_4_int -> cell_type (0:double, 1:float) + if ('sparse'): 1_4_int: number of mapped dimensions -> 'n_mapped' 'n_mapped' times: (sorted by dimension name) @@ -33,10 +43,16 @@ else: 'n_mapped' times: small_string: dimension label (same order as dimension names) prod('size_i') times: (product of all indexed dimension sizes) - double: cell value (last indexed dimension is nested innermost) + cell_type: cell value (last indexed dimension is nested innermost) //----------------------------------------------------------------------------- +Note: The mixed_with_cell_type format can be used to encode any +tensor. + +Note: cell_type defaults to double, but can be overridden by using any +of the '_with_cell_type' formats. + Note: A tensor with no dimensions should not be serialized as sparse[1], but when it is, it will contain an integer indicating the number of cells. diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java index 574be20e910..1e0d90e71f9 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/application/BindingRepository.java @@ -34,7 +34,7 @@ public class BindingRepository<T> implements Iterable<Map.Entry<UriPattern, T>> */ public void bind(String uriPattern, T target) { if (uriPattern.startsWith("https://")) { - log.warning(() -> "Bindings with 'https' scheme are deprecated. Use 'http' to match both 'http' and 'https' in URIs."); + log.warning(() -> String.format("For binding '%s': 'https' is deprecated, use 'http' to match both 'http' and 'https'", uriPattern)); } put(new UriPattern(uriPattern), target); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index 5c3298a7aff..9a10c70ceab 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -87,6 +87,14 @@ public class AccessLogRequestLog extends AbstractLifeCycle implements RequestLog if (clientCert != null && clientCert.length > 0) { accessLogEntry.setSslPrincipal(clientCert[0].getSubjectX500Principal()); } + String sslSessionId = (String) request.getAttribute(ServletRequest.SERVLET_REQUEST_SSL_SESSION_ID); + if (sslSessionId != null) { + accessLogEntry.addKeyValue("ssl-session-id", sslSessionId); + } + String cipherSuite = (String) request.getAttribute(ServletRequest.SERVLET_REQUEST_CIPHER_SUITE); + if (cipherSuite != null) { + accessLogEntry.addKeyValue("cipher-suite", cipherSuite); + } final long startTime = request.getTimeStamp(); final long endTime = System.currentTimeMillis(); diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java index 2eb7f432ec2..65c8e153164 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/servlet/ServletRequest.java @@ -39,6 +39,8 @@ public class ServletRequest extends HttpServletRequestWrapper implements Servlet public static final String JDISC_REQUEST_PRINCIPAL = "jdisc.request.principal"; public static final String JDISC_REQUEST_X509CERT = "jdisc.request.X509Certificate"; public static final String SERVLET_REQUEST_X509CERT = "javax.servlet.request.X509Certificate"; + public static final String SERVLET_REQUEST_SSL_SESSION_ID = "javax.servlet.request.ssl_session_id"; + public static final String SERVLET_REQUEST_CIPHER_SUITE = "javax.servlet.request.cipher_suite"; private final HttpServletRequest request; private final HeaderFields headerFields; diff --git a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java index d36457cf42d..704a73710cf 100644 --- a/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java +++ b/jdisc_messagebus_service/src/main/java/com/yahoo/messagebus/shared/SharedMessageBus.java @@ -4,7 +4,11 @@ package com.yahoo.messagebus.shared; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.jdisc.AbstractResource; import com.yahoo.log.LogLevel; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.DestinationSessionParams; +import com.yahoo.messagebus.IntermediateSessionParams; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.MessageBusParams; +import com.yahoo.messagebus.SourceSessionParams; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.rpc.RPCNetwork; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; diff --git a/logd/src/apps/retention/retention-enforcer.sh b/logd/src/apps/retention/retention-enforcer.sh index d1defacd4e3..a8f3730f4ce 100755 --- a/logd/src/apps/retention/retention-enforcer.sh +++ b/logd/src/apps/retention/retention-enforcer.sh @@ -15,11 +15,15 @@ RETAIN_DAYS=30 # The "database" holds lines with format "timestamp /path/to/logfile" # where "timestamp" is just seconds since epoch. +log_msg() { + echo "$0 [$$]" "$@" +} + prereq_dir() { if [ -d $1 ] && [ -w $1 ]; then : else - echo "$0: missing directory '$1' in '`pwd`'" >&2 + log_msg ERROR "missing directory '$1' in '`pwd`'" >&2 exit 1 fi } @@ -33,13 +37,14 @@ ensure_dir () { if [ -d $1 ] && [ -w $1 ]; then return 0 fi - echo "Creating directory '$1' in '`pwd`'" + log_msg INFO "Creating directory '$1' in '`pwd`'" mkdir -p $1 || exit 1 } prepare_stuff() { check_prereqs - exec > $DBGF.$$.log 2>&1 + ensure_dir ${DBGF%/*} + exec >> $DBGF.log 2>&1 ensure_dir $DBDIR } @@ -68,7 +73,7 @@ check_pidfile() { ps -p $pid >/dev/null 2>&1 || return 1 proc=$(ps -p $pid 2>&1) case $proc in *retention*) ;; *) return 1;; esac - echo "$0 [$$]: Yielding my place to pid '$pid'" + log_msg INFO "Yielding my place to pid '$pid'" exit 1 fi } @@ -82,7 +87,7 @@ maybe_collect() { logfilename=$2 if bad_timestamp "$1"; then - echo "WARNING: bad timestamp '$timestamp' for logfilename '$logfilename'" + log_msg WARNING "bad timestamp '$timestamp' for logfilename '$logfilename'" return fi @@ -92,7 +97,7 @@ maybe_collect() { lim2=$(($mod_time + $add)) if [ $lim1 -lt $now ] && [ $lim2 -lt $now ]; then - echo "Collect logfile '$logfilename' timestamped $timestamp modified $mod_time" + log_msg INFO "Collect logfile '$logfilename' timestamped $timestamp modified $mod_time" rm -f "$logfilename" fi } @@ -124,10 +129,10 @@ process_all() { mainloop() { while true; do + check_pidfile mark_pid process_all sleep 3600 - check_pidfile done } diff --git a/logd/src/logd/config_subscriber.cpp b/logd/src/logd/config_subscriber.cpp index 4a5bfc8bcb9..ce29c808742 100644 --- a/logd/src/logd/config_subscriber.cpp +++ b/logd/src/logd/config_subscriber.cpp @@ -141,7 +141,7 @@ ConfigSubscriber::make_forwarder(Metrics& metrics) result = LegacyForwarder::to_logserver(metrics, _forward_filter, _logserver_host, _logserver_port); } } else { - LegacyForwarder::to_dev_null(metrics); + result = LegacyForwarder::to_dev_null(metrics); } _need_new_forwarder = false; return result; diff --git a/messagebus/abi-spec.json b/messagebus/abi-spec.json index 288bd6a1eda..e8e752127b1 100644 --- a/messagebus/abi-spec.json +++ b/messagebus/abi-spec.json @@ -576,7 +576,6 @@ ], "methods": [ "public void <init>()", - "public void addRecurrentTask(com.yahoo.messagebus.Messenger$Task)", "public void start()", "public void deliverMessage(com.yahoo.messagebus.Message, com.yahoo.messagebus.MessageHandler)", "public void deliverReply(com.yahoo.messagebus.Reply, com.yahoo.messagebus.ReplyHandler)", diff --git a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java index 26e61e8917b..0f0b704bba7 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/MessageBus.java @@ -58,13 +58,13 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, private static Logger log = Logger.getLogger(MessageBus.class.getName()); private final AtomicBoolean destroyed = new AtomicBoolean(false); private final ProtocolRepository protocolRepository = new ProtocolRepository(); - private final AtomicReference<Map<String, RoutingTable>> tablesRef = new AtomicReference<Map<String, RoutingTable>>(null); - private final CopyOnWriteHashMap<String, MessageHandler> sessions = new CopyOnWriteHashMap<String, MessageHandler>(); + private final AtomicReference<Map<String, RoutingTable>> tablesRef = new AtomicReference<>(null); + private final CopyOnWriteHashMap<String, MessageHandler> sessions = new CopyOnWriteHashMap<>(); private final Network net; private final Messenger msn; private final Resender resender; - private int maxPendingCount = 0; - private int maxPendingSize = 0; + private int maxPendingCount; + private int maxPendingSize; private int pendingCount = 0; private int pendingSize = 0; private final Thread careTaker = new Thread(this::sendBlockedMessages); @@ -440,7 +440,7 @@ public class MessageBus implements ConfigHandler, NetworkOwner, MessageHandler, @Override public void setupRouting(RoutingSpec spec) { - Map<String, RoutingTable> tables = new HashMap<String, RoutingTable>(); + Map<String, RoutingTable> tables = new HashMap<>(); for (int i = 0, len = spec.getNumTables(); i < len; ++i) { RoutingTableSpec table = spec.getTable(i); String name = table.getProtocol(); diff --git a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java index 4fb83386231..63e5dbb2d04 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/Messenger.java @@ -44,7 +44,7 @@ public class Messenger implements Runnable { * * @param task The task to add. */ - public void addRecurrentTask(final Task task) { + void addRecurrentTask(final Task task) { children.add(task); } @@ -159,7 +159,7 @@ public class Messenger implements Runnable { synchronized (this) { if (queue.isEmpty()) { try { - wait(100); + wait(10); } catch (final InterruptedException e) { continue; } @@ -210,13 +210,13 @@ public class Messenger implements Runnable { /** * <p>This method is called when being executed.</p> */ - public void run(); + void run(); /** * <p>This method is called for all tasks, even if {@link #run()} was * never called.</p> */ - public void destroy(); + void destroy(); } private static class MessageTask implements Runnable { diff --git a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java index 64b7c4cb12e..e04cccfcbd1 100755 --- a/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/routing/RoutingNode.java @@ -1,15 +1,28 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.routing; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBus; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.ReplyHandler; +import com.yahoo.messagebus.SendProxy; +import com.yahoo.messagebus.Trace; +import com.yahoo.messagebus.TraceLevel; +import com.yahoo.messagebus.TraceNode; import com.yahoo.messagebus.metrics.RouteMetricSet; import com.yahoo.messagebus.network.Network; import com.yahoo.messagebus.network.ServiceAddress; import java.io.PrintWriter; import java.io.StringWriter; -import java.util.*; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.Stack; import java.util.concurrent.atomic.AtomicInteger; /** @@ -32,7 +45,7 @@ public class RoutingNode implements ReplyHandler { private final AtomicInteger pending = new AtomicInteger(0); private final Message msg; private Reply reply = null; - private Route route = null; + private Route route; private RoutingPolicy policy = null; private RoutingContext routingContext = null; private ServiceAddress serviceAddress = null; @@ -122,7 +135,7 @@ public class RoutingNode implements ReplyHandler { * * @param msg The error message to assign. */ - public void notifyAbort(String msg) { + private void notifyAbort(String msg) { Stack<RoutingNode> stack = new Stack<>(); stack.push(this); while (!stack.isEmpty()) { diff --git a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java index 0ac962ad947..2ebe4103713 100644 --- a/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java +++ b/messagebus/src/test/java/com/yahoo/messagebus/routing/RoutingTestCase.java @@ -4,8 +4,17 @@ package com.yahoo.messagebus.routing; import com.yahoo.component.Vtag; import com.yahoo.jrt.ListenFailedException; import com.yahoo.jrt.slobrok.server.Slobrok; -import com.yahoo.messagebus.*; +import com.yahoo.messagebus.DestinationSession; +import com.yahoo.messagebus.DestinationSessionParams; +import com.yahoo.messagebus.EmptyReply; import com.yahoo.messagebus.Error; +import com.yahoo.messagebus.ErrorCode; +import com.yahoo.messagebus.Message; +import com.yahoo.messagebus.MessageBusParams; +import com.yahoo.messagebus.Reply; +import com.yahoo.messagebus.SourceSession; +import com.yahoo.messagebus.SourceSessionParams; +import com.yahoo.messagebus.Trace; import com.yahoo.messagebus.network.Identity; import com.yahoo.messagebus.network.rpc.RPCNetworkParams; import com.yahoo.messagebus.network.rpc.test.TestServer; @@ -23,7 +32,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; /** * @author <a href="mailto:havardpe@yahoo-inc.com">Haavard Pettersen</a> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java index 10ac30d8715..1811fc0c8f0 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/component/ConfigServerInfo.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.component; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import java.net.URI; @@ -16,12 +17,12 @@ import java.util.stream.Collectors; */ public class ConfigServerInfo { private final URI loadBalancerEndpoint; - private final AthenzService configServerIdentity; + private final AthenzIdentity configServerIdentity; private final Function<String, URI> configServerHostnameToUriMapper; private final List<URI> configServerURIs; public ConfigServerInfo(String loadBalancerHostName, List<String> configServerHostNames, - String scheme, int port, AthenzService configServerAthenzIdentity) { + String scheme, int port, AthenzIdentity configServerAthenzIdentity) { this.loadBalancerEndpoint = createLoadBalancerEndpoint(loadBalancerHostName, scheme, port); this.configServerIdentity = configServerAthenzIdentity; this.configServerHostnameToUriMapper = hostname -> URI.create(scheme + "://" + hostname + ":" + port); @@ -46,7 +47,7 @@ public class ConfigServerInfo { return loadBalancerEndpoint; } - public AthenzService getConfigServerIdentity() { + public AthenzIdentity getConfigServerIdentity() { return configServerIdentity; } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 4fe0f420f05..550d6e7021e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -8,6 +8,7 @@ import com.yahoo.security.KeyUtils; import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.DefaultZtsClient; import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; @@ -63,7 +64,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private final URI ztsEndpoint; private final Path trustStorePath; - private final AthenzService configserverIdentity; + private final AthenzIdentity configserverIdentity; private final Clock clock; private final ServiceIdentityProvider hostIdentityProvider; private final IdentityDocumentClient identityDocumentClient; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java index 496f4bd667d..205e7b1e258 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.nodeagent; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; @@ -33,7 +34,7 @@ public interface NodeAgentContext extends TaskContext { return node().getNodeType(); } - AthenzService identity(); + AthenzIdentity identity(); DockerNetworking dockerNetworking(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index 9ca19a76706..1b33fed151e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -4,6 +4,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.dockerapi.ContainerName; import com.yahoo.vespa.hosted.node.admin.component.ZoneId; @@ -30,7 +31,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { private final NodeSpec node; private final Acl acl; private final ContainerName containerName; - private final AthenzService identity; + private final AthenzIdentity identity; private final DockerNetworking dockerNetworking; private final ZoneId zoneId; private final Path pathToNodeRootOnHost; @@ -38,7 +39,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { private final String vespaUser; private final String vespaUserOnHost; - public NodeAgentContextImpl(NodeSpec node, Acl acl, AthenzService identity, + public NodeAgentContextImpl(NodeSpec node, Acl acl, AthenzIdentity identity, DockerNetworking dockerNetworking, ZoneId zoneId, Path pathToContainerStorage, Path pathToVespaHome, String vespaUser, String vespaUserOnHost) { @@ -71,7 +72,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { } @Override - public AthenzService identity() { + public AthenzIdentity identity() { return identity; } @@ -157,7 +158,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { public static class Builder { private NodeSpec.Builder nodeSpecBuilder = new NodeSpec.Builder(); private Acl acl; - private AthenzService identity; + private AthenzIdentity identity; private DockerNetworking dockerNetworking; private ZoneId zoneId; private Path pathToContainerStorage; @@ -192,7 +193,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { return this; } - public Builder identity(AthenzService identity) { + public Builder identity(AthenzIdentity identity) { this.identity = identity; return this; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java index 7fdd9a168c8..3b5f9d91725 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintenance.java @@ -180,7 +180,7 @@ public class NodeRepositoryMaintenance extends AbstractComponent { private final NodeFailer.ThrottlePolicy throttlePolicy; DefaultTimes(Zone zone) { - failGrace = Duration.ofMinutes(60); + failGrace = Duration.ofMinutes(30); periodicRedeployInterval = Duration.ofMinutes(30); // Don't redeploy in test environments redeployMaintainerInterval = zone.environment().isTest() ? Duration.ofDays(1) : Duration.ofMinutes(1); diff --git a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp index 18d4fbdb6d7..b2334ed025e 100644 --- a/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp +++ b/searchcore/src/tests/proton/feed_and_search/feed_and_search.cpp @@ -5,7 +5,7 @@ LOG_SETUP("feed_and_search_test"); #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/fieldvalue/fieldvalue.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/diskindex/diskindex.h> #include <vespa/searchlib/diskindex/indexbuilder.h> #include <vespa/searchlib/fef/fef.h> diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp index be41aa96efd..9a1ddee4278 100644 --- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp +++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp @@ -4,7 +4,7 @@ #include <vespa/searchcore/proton/index/indexmanager.h> #include <vespa/searchcore/proton/server/executorthreadingservice.h> #include <vespa/searchcorespi/index/fusionrunner.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/diskindex/diskindex.h> #include <vespa/searchlib/diskindex/indexbuilder.h> #include <vespa/searchlib/fef/matchdatalayout.h> diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 82a3590b6b9..841c69289bf 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -14,11 +14,10 @@ #include <vespa/searchlib/common/serialnum.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/memoryindex/compact_document_words_store.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> +#include <vespa/searchlib/memoryindex/compact_words_store.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> #include <vespa/searchlib/memoryindex/field_index_collection.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> #include <vespa/searchlib/queryeval/isourceselector.h> #include <vespa/searchlib/util/dirtraverse.h> #include <vespa/vespalib/io/fileutil.h> @@ -43,7 +42,7 @@ using search::index::DummyFileHeaderContext; using search::index::Schema; using search::index::schema::DataType; using vespalib::makeLambdaTask; -using search::memoryindex::CompactDocumentWordsStore; +using search::memoryindex::CompactWordsStore; using search::memoryindex::FieldIndexCollection; using search::queryeval::Source; using std::set; diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h index 6124eadb34d..2ca6f969c55 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchcorespi/index/imemoryindex.h> #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/searchlib/common/tunefileinfo.h> diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 6d1695155a8..66408b1d7d7 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -173,14 +173,14 @@ vespa_define_module( src/tests/index/doctypebuilder src/tests/indexmetainfo src/tests/ld-library-path - src/tests/memoryindex/compact_document_words_store + src/tests/memoryindex/compact_words_store src/tests/memoryindex/datastore - src/tests/memoryindex/document_remover - src/tests/memoryindex/documentinverter + src/tests/memoryindex/document_inverter src/tests/memoryindex/field_index - src/tests/memoryindex/fieldinverter - src/tests/memoryindex/memoryindex - src/tests/memoryindex/urlfieldinverter + src/tests/memoryindex/field_index_remover + src/tests/memoryindex/field_inverter + src/tests/memoryindex/memory_index + src/tests/memoryindex/url_field_inverter src/tests/nativerank src/tests/nearsearch src/tests/postinglistbm diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index b8c51f4e33d..0b9cb06d2a5 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1382,7 +1382,7 @@ "public void <init>(java.util.Map)", "public void <init>(java.util.Map, java.util.Map)", "public com.yahoo.searchlib.rankingexpression.ExpressionFunction getFunction(java.lang.String)", - "protected java.util.Map functions()", + "protected final com.google.common.collect.ImmutableMap functions()", "public java.lang.String getBinding(java.lang.String)", "public com.yahoo.searchlib.rankingexpression.rule.FunctionReferenceContext withBindings(java.util.Map)" ], diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index b911284a1b4..2ef8448db8b 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> diff --git a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java index d880294660e..200e4fbe856 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java @@ -19,9 +19,9 @@ public final class FieldMatchMetrics implements Cloneable { private FieldMatchMetricsComputer source; /** The trace accumulated during execution - empty if no tracing */ - private final Trace trace=new Trace(); + private final Trace trace = new Trace(); - private boolean complete=false; + private boolean complete; // Metrics private int outOfOrder; @@ -352,7 +352,7 @@ public final class FieldMatchMetrics implements Cloneable { * </p> * * - * <p>Weight and significance are not taken into account because this is mean to capture tha quality of the + * <p>Weight and significance are not taken into account because this is meant to capture tha quality of the * match in this field, while those measures relate this match to matches in other fields. This number * can be multiplied with those values when combining with other field match scores.</p> */ diff --git a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetricsComputer.java b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetricsComputer.java index 79886449d0a..f981ad464ec 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetricsComputer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetricsComputer.java @@ -136,7 +136,7 @@ public final class FieldMatchMetricsComputer { // Explore segmentations while (segmentStartPoint!=null) { - metrics =segmentStartPoint.getMetrics().clone(); + metrics = segmentStartPoint.getMetrics().clone(); if (collectTrace) metrics.trace().add("\nLooking for segment from " + segmentStartPoint + "..." + "\n"); boolean found=findAlternativeSegmentFrom(segmentStartPoint); @@ -148,7 +148,7 @@ public final class FieldMatchMetricsComputer { segmentStartPoint=findOpenSegment(segmentStartPoint.getI()); } - metrics=findLastStartPoint().getMetrics(); // these metrics are the final set + metrics = findLastStartPoint().getMetrics(); // these metrics are the final set setOccurrenceCounts(metrics); metrics.onComplete(); metrics.setComplete(true); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java index ed1e2838717..084bfe65e06 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionReferenceContext.java @@ -61,7 +61,7 @@ public class FunctionReferenceContext { */ public ExpressionFunction getFunction(String name) { return functions.get(name); } - protected Map<String, ExpressionFunction> functions() { return functions; } + protected final ImmutableMap<String, ExpressionFunction> functions() { return functions; } /** Returns the resolution of an argument, or null if it isn't defined in this context */ public String getBinding(String name) { return bindings.get(name); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index 0b68e71c21a..4acc1a85490 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -3,12 +3,10 @@ package com.yahoo.searchlib.rankingexpression.rule; import com.google.common.collect.ImmutableMap; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; -import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.tensor.TensorType; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -95,7 +93,7 @@ public class SerializationContext extends FunctionReferenceContext { @Override public SerializationContext withBindings(Map<String, String> bindings) { - return new SerializationContext(functions().values(), bindings, this.serializedFunctions); + return new SerializationContext(functions(), bindings, this.serializedFunctions); } public Map<String, String> serializedFunctions() { return serializedFunctions; } diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp index fb6535c4a70..24146e516a0 100644 --- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp +++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp @@ -12,9 +12,9 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> #include <vespa/searchlib/memoryindex/field_index_collection.h> -#include <vespa/searchlib/memoryindex/postingiterator.h> +#include <vespa/searchlib/memoryindex/posting_iterator.h> #include <vespa/searchlib/util/filekit.h> #include <vespa/vespalib/testkit/testapp.h> diff --git a/searchlib/src/tests/memoryindex/compact_document_words_store/.gitignore b/searchlib/src/tests/memoryindex/compact_document_words_store/.gitignore deleted file mode 100644 index 3ad290f1731..00000000000 --- a/searchlib/src/tests/memoryindex/compact_document_words_store/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_compact_document_words_store_test_app diff --git a/searchlib/src/tests/memoryindex/compact_document_words_store/CMakeLists.txt b/searchlib/src/tests/memoryindex/compact_document_words_store/CMakeLists.txt deleted file mode 100644 index a5c8e0b2d14..00000000000 --- a/searchlib/src/tests/memoryindex/compact_document_words_store/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_compact_document_words_store_test_app TEST - SOURCES - compact_document_words_store_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_compact_document_words_store_test_app COMMAND searchlib_compact_document_words_store_test_app) diff --git a/searchlib/src/tests/memoryindex/compact_words_store/.gitignore b/searchlib/src/tests/memoryindex/compact_words_store/.gitignore new file mode 100644 index 00000000000..9f9acb50adc --- /dev/null +++ b/searchlib/src/tests/memoryindex/compact_words_store/.gitignore @@ -0,0 +1 @@ +searchlib_compact_words_store_test_app diff --git a/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt b/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt new file mode 100644 index 00000000000..ee31ef7c7aa --- /dev/null +++ b/searchlib/src/tests/memoryindex/compact_words_store/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_compact_words_store_test_app TEST + SOURCES + compact_words_store_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_compact_words_store_test_app COMMAND searchlib_compact_words_store_test_app) diff --git a/searchlib/src/tests/memoryindex/compact_document_words_store/compact_document_words_store_test.cpp b/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp index 6e22a4e5ff6..04d66396f90 100644 --- a/searchlib/src/tests/memoryindex/compact_document_words_store/compact_document_words_store_test.cpp +++ b/searchlib/src/tests/memoryindex/compact_words_store/compact_words_store_test.cpp @@ -2,7 +2,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/searchlib/datastore/entryref.h> -#include <vespa/searchlib/memoryindex/compact_document_words_store.h> +#include <vespa/searchlib/memoryindex/compact_words_store.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <iostream> @@ -12,8 +12,8 @@ using namespace search; using namespace search::datastore; using namespace search::memoryindex; -typedef CompactDocumentWordsStore::Builder Builder; -typedef CompactDocumentWordsStore::Iterator Iterator; +typedef CompactWordsStore::Builder Builder; +typedef CompactWordsStore::Iterator Iterator; typedef Builder::WordRefVector WordRefVector; const EntryRef w1(1); @@ -53,7 +53,7 @@ toStr(Iterator itr) struct SingleFixture { - CompactDocumentWordsStore _store; + CompactWordsStore _store; SingleFixture() : _store() { _store.insert(Builder(d1).insert(w1).insert(w2).insert(w3)); } @@ -61,7 +61,7 @@ struct SingleFixture struct MultiFixture { - CompactDocumentWordsStore _store; + CompactWordsStore _store; MultiFixture() : _store() { _store.insert(Builder(d1).insert(w1)); _store.insert(Builder(d2).insert(w2)); @@ -100,7 +100,7 @@ TEST_F("require that documents can be removed and re-inserted", MultiFixture) TEST("require that a lot of words can be inserted, retrieved and removed") { - CompactDocumentWordsStore store; + CompactWordsStore store; for (uint32_t docId = 0; docId < 50; ++docId) { Builder b(docId); for (uint32_t wordRef = 0; wordRef < 20000; ++wordRef) { @@ -125,9 +125,9 @@ TEST("require that a lot of words can be inserted, retrieved and removed") TEST("require that initial memory usage is reported") { - CompactDocumentWordsStore store; - CompactDocumentWordsStore::DocumentWordsMap docs; - CompactDocumentWordsStore::Store internalStore; + CompactWordsStore store; + CompactWordsStore::DocumentWordsMap docs; + CompactWordsStore::Store internalStore; MemoryUsage initExp; initExp.incAllocatedBytes(docs.getMemoryConsumption()); initExp.incUsedBytes(docs.getMemoryUsed()); @@ -142,7 +142,7 @@ TEST("require that initial memory usage is reported") TEST("require that memory usage is updated after insert") { - CompactDocumentWordsStore store; + CompactWordsStore store; MemoryUsage init = store.getMemoryUsage(); store.insert(Builder(d1).insert(w1)); diff --git a/searchlib/src/tests/memoryindex/datastore/.gitignore b/searchlib/src/tests/memoryindex/datastore/.gitignore index 98f4acc70a8..a5842a0fd69 100644 --- a/searchlib/src/tests/memoryindex/datastore/.gitignore +++ b/searchlib/src/tests/memoryindex/datastore/.gitignore @@ -1,8 +1,4 @@ .depend Makefile -datastore_test -featurestore_test -wordstore_test -searchlib_datastore_test_app -searchlib_featurestore_test_app -searchlib_wordstore_test_app +searchlib_feature_store_test_app +searchlib_word_store_test_app diff --git a/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt index 2ba0f2eac63..45507f3b0ae 100644 --- a/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt +++ b/searchlib/src/tests/memoryindex/datastore/CMakeLists.txt @@ -1,15 +1,15 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_featurestore_test_app TEST +vespa_add_executable(searchlib_feature_store_test_app TEST SOURCES - featurestore_test.cpp + feature_store_test.cpp DEPENDS searchlib ) -vespa_add_test(NAME searchlib_featurestore_test_app COMMAND searchlib_featurestore_test_app) -vespa_add_executable(searchlib_wordstore_test_app TEST +vespa_add_test(NAME searchlib_feature_store_test_app COMMAND searchlib_feature_store_test_app) +vespa_add_executable(searchlib_word_store_test_app TEST SOURCES - wordstore_test.cpp + word_store_test.cpp DEPENDS searchlib ) -vespa_add_test(NAME searchlib_wordstore_test_app COMMAND searchlib_wordstore_test_app) +vespa_add_test(NAME searchlib_word_store_test_app COMMAND searchlib_word_store_test_app) diff --git a/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp index dc061f55732..49e9d613861 100644 --- a/searchlib/src/tests/memoryindex/datastore/featurestore_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> -LOG_SETUP("featurestore_test"); +LOG_SETUP("feature_store_test"); #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/featurestore.h> +#include <vespa/searchlib/memoryindex/feature_store.h> using namespace search::btree; using namespace search::datastore; @@ -213,7 +213,7 @@ Test::Test() int Test::Main() { - TEST_INIT("featurestore_test"); + TEST_INIT("feature_store_test"); requireThatFeaturesCanBeAddedAndRetrieved(); requireThatNextWordsAreWorking(); diff --git a/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp b/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp index c1baff72514..b7f454bfdf7 100644 --- a/searchlib/src/tests/memoryindex/datastore/wordstore_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/word_store_test.cpp @@ -1,8 +1,8 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> -LOG_SETUP("wordstore_test"); +LOG_SETUP("word_store_test"); #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/wordstore.h> +#include <vespa/searchlib/memoryindex/word_store.h> using namespace search::datastore; @@ -75,7 +75,7 @@ Test::requireThatAddWordTriggersChangeOfBuffer() int Test::Main() { - TEST_INIT("wordstore_test"); + TEST_INIT("word_store_test"); requireThatWordsCanBeAddedAndRetrieved(); requireThatAddWordTriggersChangeOfBuffer(); diff --git a/searchlib/src/tests/memoryindex/document_inverter/.gitignore b/searchlib/src/tests/memoryindex/document_inverter/.gitignore new file mode 100644 index 00000000000..6245bb146e7 --- /dev/null +++ b/searchlib/src/tests/memoryindex/document_inverter/.gitignore @@ -0,0 +1 @@ +searchlib_document_inverter_test_app diff --git a/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt new file mode 100644 index 00000000000..1058a19d0ce --- /dev/null +++ b/searchlib/src/tests/memoryindex/document_inverter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_document_inverter_test_app TEST + SOURCES + document_inverter_test.cpp + DEPENDS + searchlib_test + searchlib +) +vespa_add_test(NAME searchlib_document_inverter_test_app COMMAND searchlib_document_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp index 36cd15c8ada..91c1ccba706 100644 --- a/searchlib/src/tests/memoryindex/documentinverter/documentinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/document_inverter/document_inverter_test.cpp @@ -3,9 +3,9 @@ #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> #include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/vespalib/testkit/testapp.h> @@ -97,7 +97,7 @@ struct Fixture SequencedTaskExecutor _invertThreads; SequencedTaskExecutor _pushThreads; DocumentInverter _inv; - test::OrderedDocumentInserter _inserter; + test::OrderedFieldIndexInserter _inserter; static Schema makeSchema() diff --git a/searchlib/src/tests/memoryindex/document_remover/CMakeLists.txt b/searchlib/src/tests/memoryindex/document_remover/CMakeLists.txt deleted file mode 100644 index 04e5b0ec126..00000000000 --- a/searchlib/src/tests/memoryindex/document_remover/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_document_remover_test_app TEST - SOURCES - document_remover_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_document_remover_test_app COMMAND searchlib_document_remover_test_app) diff --git a/searchlib/src/tests/memoryindex/documentinverter/.gitignore b/searchlib/src/tests/memoryindex/documentinverter/.gitignore deleted file mode 100644 index 1e9666b2d63..00000000000 --- a/searchlib/src/tests/memoryindex/documentinverter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_documentinverter_test_app diff --git a/searchlib/src/tests/memoryindex/documentinverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/documentinverter/CMakeLists.txt deleted file mode 100644 index c7760e9f12f..00000000000 --- a/searchlib/src/tests/memoryindex/documentinverter/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_documentinverter_test_app TEST - SOURCES - documentinverter_test.cpp - DEPENDS - searchlib_test - searchlib -) -vespa_add_test(NAME searchlib_documentinverter_test_app COMMAND searchlib_documentinverter_test_app) diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index 373ed7fd311..3a635756ec7 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -10,11 +10,11 @@ #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> -#include <vespa/searchlib/memoryindex/documentinverter.h> +#include <vespa/searchlib/memoryindex/document_inverter.h> #include <vespa/searchlib/memoryindex/field_index_collection.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/memoryindex/ordereddocumentinserter.h> -#include <vespa/searchlib/memoryindex/postingiterator.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/memoryindex/ordered_field_index_inserter.h> +#include <vespa/searchlib/memoryindex/posting_iterator.h> #include <vespa/searchlib/test/searchiteratorverifier.h> #include <vespa/vespalib/testkit/testapp.h> @@ -300,7 +300,7 @@ MockFieldIndex::~MockFieldIndex() = default; /** * MockWordStoreScan is a helper class to ensure that previous word is - * still stored safely in memory, to satisfy OrderedDocumentInserter + * still stored safely in memory, to satisfy OrderedFieldIndexInserter * needs. */ class MockWordStoreScan @@ -347,7 +347,7 @@ class MyInserter MockFieldIndex _mock; FieldIndexCollection _fieldIndexes; DocIdAndPosOccFeatures _features; - IOrderedDocumentInserter *_documentInserter; + IOrderedFieldIndexInserter *_inserter; public: MyInserter(const Schema &schema) @@ -355,7 +355,7 @@ public: _mock(), _fieldIndexes(schema), _features(), - _documentInserter(nullptr) + _inserter(nullptr) { _features.addNextOcc(0, 0, 1, 1); } @@ -365,32 +365,32 @@ public: setNextWord(const vespalib::string &word) { const vespalib::string &w = _wordStoreScan.setWord(word); - _documentInserter->setNextWord(w); + _inserter->setNextWord(w); _mock.setNextWord(w); } void setNextField(uint32_t fieldId) { - if (_documentInserter != nullptr) { - _documentInserter->flush(); + if (_inserter != nullptr) { + _inserter->flush(); } - _documentInserter = &_fieldIndexes.getFieldIndex(fieldId)->getInserter(); - _documentInserter->rewind(); + _inserter = &_fieldIndexes.getFieldIndex(fieldId)->getInserter(); + _inserter->rewind(); _mock.setNextField(fieldId); } void add(uint32_t docId) { - _documentInserter->add(docId, _features); + _inserter->add(docId, _features); _mock.add(docId); } void remove(uint32_t docId) { - _documentInserter->remove(docId); + _inserter->remove(docId); _mock.remove(docId); } @@ -406,8 +406,8 @@ public: bool assertPostings() { - if (_documentInserter != nullptr) { - _documentInserter->flush(); + if (_inserter != nullptr) { + _inserter->flush(); } for (auto wfp : _mock) { auto &wf = wfp.first; @@ -423,9 +423,9 @@ public: void rewind() { - if (_documentInserter != nullptr) { - _documentInserter->flush(); - _documentInserter = nullptr; + if (_inserter != nullptr) { + _inserter->flush(); + _inserter = nullptr; } } @@ -451,7 +451,7 @@ myremove(uint32_t docId, DocumentInverter &inv, FieldIndexCollection &fieldIndex class WrapInserter { - OrderedDocumentInserter &_inserter; + OrderedFieldIndexInserter &_inserter; public: WrapInserter(FieldIndexCollection &fieldIndexes, uint32_t fieldId) : _inserter(fieldIndexes.getFieldIndex(fieldId)->getInserter()) @@ -503,9 +503,9 @@ public: }; -class MyDrainRemoves : IDocumentRemoveListener +class MyDrainRemoves : IFieldIndexRemoveListener { - DocumentRemover &_remover; + FieldIndexRemover &_remover; public: virtual void remove(const vespalib::stringref, uint32_t) override { } diff --git a/searchlib/src/tests/memoryindex/document_remover/.gitignore b/searchlib/src/tests/memoryindex/field_index_remover/.gitignore index 2126f9147bd..2126f9147bd 100644 --- a/searchlib/src/tests/memoryindex/document_remover/.gitignore +++ b/searchlib/src/tests/memoryindex/field_index_remover/.gitignore diff --git a/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt new file mode 100644 index 00000000000..ef75337c6b6 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_index_remover/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_field_index_remover_test_app TEST + SOURCES + field_index_remover_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_field_index_remover_test_app COMMAND searchlib_field_index_remover_test_app) diff --git a/searchlib/src/tests/memoryindex/document_remover/document_remover_test.cpp b/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp index af7a9422e49..fed6d963b70 100644 --- a/searchlib/src/tests/memoryindex/document_remover/document_remover_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index_remover/field_index_remover_test.cpp @@ -2,9 +2,9 @@ #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/document_remover.h> -#include <vespa/searchlib/memoryindex/wordstore.h> -#include <vespa/searchlib/memoryindex/i_document_remove_listener.h> +#include <vespa/searchlib/memoryindex/field_index_remover.h> +#include <vespa/searchlib/memoryindex/i_field_index_remove_listener.h> +#include <vespa/searchlib/memoryindex/word_store.h> #include <vespa/vespalib/test/insertion_operators.h> #include <algorithm> @@ -38,7 +38,7 @@ operator<<(std::ostream &os, const WordFieldPair &val) return os; } -struct MockRemoveListener : public IDocumentRemoveListener +struct MockRemoveListener : public IFieldIndexRemoveListener { WordFieldVector _words; uint32_t _expDocId; @@ -65,7 +65,7 @@ struct Fixture MockRemoveListener _listener; std::vector<std::unique_ptr<WordStore>> _wordStores; std::vector<std::map<vespalib::string, datastore::EntryRef>> _wordToRefMaps; - std::vector<std::unique_ptr<DocumentRemover>> _removers; + std::vector<std::unique_ptr<FieldIndexRemover>> _removers; Fixture() : _listener(), _wordStores(), @@ -75,7 +75,7 @@ struct Fixture uint32_t numFields = 4; for (uint32_t fieldId = 0; fieldId < numFields; ++fieldId) { _wordStores.push_back(std::make_unique<WordStore>()); - _removers.push_back(std::make_unique<DocumentRemover> + _removers.push_back(std::make_unique<FieldIndexRemover> (*_wordStores.back())); } _wordToRefMaps.resize(numFields); diff --git a/searchlib/src/tests/memoryindex/field_inverter/.gitignore b/searchlib/src/tests/memoryindex/field_inverter/.gitignore new file mode 100644 index 00000000000..58579d09421 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_inverter/.gitignore @@ -0,0 +1 @@ +searchlib_field_inverter_test_app diff --git a/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt new file mode 100644 index 00000000000..f39e05d6823 --- /dev/null +++ b/searchlib/src/tests/memoryindex/field_inverter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_field_inverter_test_app TEST + SOURCES + field_inverter_test.cpp + DEPENDS + searchlib_test + searchlib +) +vespa_add_test(NAME searchlib_field_inverter_test_app COMMAND searchlib_field_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp b/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp index 1d066747ef8..f08e61b0da2 100644 --- a/searchlib/src/tests/memoryindex/fieldinverter/fieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/field_inverter/field_inverter_test.cpp @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/repo/fixedtyperepo.h> namespace search { @@ -105,7 +105,7 @@ struct Fixture Schema _schema; DocBuilder _b; std::vector<std::unique_ptr<FieldInverter> > _inverters; - test::OrderedDocumentInserter _inserter; + test::OrderedFieldIndexInserter _inserter; static Schema makeSchema() diff --git a/searchlib/src/tests/memoryindex/fieldinverter/.gitignore b/searchlib/src/tests/memoryindex/fieldinverter/.gitignore deleted file mode 100644 index 482663dd92e..00000000000 --- a/searchlib/src/tests/memoryindex/fieldinverter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_fieldinverter_test_app diff --git a/searchlib/src/tests/memoryindex/fieldinverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/fieldinverter/CMakeLists.txt deleted file mode 100644 index b6b223dceed..00000000000 --- a/searchlib/src/tests/memoryindex/fieldinverter/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_fieldinverter_test_app TEST - SOURCES - fieldinverter_test.cpp - DEPENDS - searchlib_test - searchlib -) -vespa_add_test(NAME searchlib_fieldinverter_test_app COMMAND searchlib_fieldinverter_test_app) diff --git a/searchlib/src/tests/memoryindex/memory_index/.gitignore b/searchlib/src/tests/memoryindex/memory_index/.gitignore new file mode 100644 index 00000000000..6ec9ccf5015 --- /dev/null +++ b/searchlib/src/tests/memoryindex/memory_index/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +searchlib_memory_index_test_app diff --git a/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt b/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt new file mode 100644 index 00000000000..d6b6691f05d --- /dev/null +++ b/searchlib/src/tests/memoryindex/memory_index/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_memory_index_test_app TEST + SOURCES + memory_index_test.cpp + DEPENDS + searchlib +) +vespa_add_test(NAME searchlib_memory_index_test_app COMMAND searchlib_memory_index_test_app) diff --git a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 9de6ac9f310..50f44074683 100644 --- a/searchlib/src/tests/memoryindex/memoryindex/memoryindex_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -2,24 +2,24 @@ #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/searchlib/memoryindex/memoryindex.h> +#include <vespa/searchlib/common/scheduletaskcallback.h> +#include <vespa/searchlib/common/sequencedtaskexecutor.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/docbuilder.h> +#include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> +#include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_searchable.h> -#include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/sequencedtaskexecutor.h> -#include <vespa/searchlib/common/scheduletaskcallback.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/log/log.h> -LOG_SETUP("memoryindex_test"); +LOG_SETUP("memory_index_test"); using document::Document; using document::FieldValue; diff --git a/searchlib/src/tests/memoryindex/memoryindex/.gitignore b/searchlib/src/tests/memoryindex/memoryindex/.gitignore deleted file mode 100644 index 174d0a494e2..00000000000 --- a/searchlib/src/tests/memoryindex/memoryindex/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.depend -Makefile -memoryindex_test -sourceselectorwriter_test -searchlib_memoryindex_test_app diff --git a/searchlib/src/tests/memoryindex/memoryindex/CMakeLists.txt b/searchlib/src/tests/memoryindex/memoryindex/CMakeLists.txt deleted file mode 100644 index 20f526e5c99..00000000000 --- a/searchlib/src/tests/memoryindex/memoryindex/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_memoryindex_test_app TEST - SOURCES - memoryindex_test.cpp - DEPENDS - searchlib -) -vespa_add_test(NAME searchlib_memoryindex_test_app COMMAND searchlib_memoryindex_test_app) diff --git a/searchlib/src/tests/memoryindex/url_field_inverter/.gitignore b/searchlib/src/tests/memoryindex/url_field_inverter/.gitignore new file mode 100644 index 00000000000..694dc947042 --- /dev/null +++ b/searchlib/src/tests/memoryindex/url_field_inverter/.gitignore @@ -0,0 +1 @@ +searchlib_url_field_inverter_test_app diff --git a/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt new file mode 100644 index 00000000000..28efc8a861e --- /dev/null +++ b/searchlib/src/tests/memoryindex/url_field_inverter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchlib_url_field_inverter_test_app TEST + SOURCES + url_field_inverter_test.cpp + DEPENDS + searchlib_test + searchlib +) +vespa_add_test(NAME searchlib_url_field_inverter_test_app COMMAND searchlib_url_field_inverter_test_app) diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp b/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp index daec09828f6..76fbf662b77 100644 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/urlfieldinverter_test.cpp +++ b/searchlib/src/tests/memoryindex/url_field_inverter/url_field_inverter_test.cpp @@ -1,13 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /* -*- mode: C++; coding: utf-8; -*- */ - +#include <vespa/document/repo/fixedtyperepo.h> #include <vespa/searchlib/index/docbuilder.h> -#include <vespa/searchlib/memoryindex/fieldinverter.h> -#include <vespa/searchlib/memoryindex/urlfieldinverter.h> -#include <vespa/searchlib/test/memoryindex/ordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/field_inverter.h> +#include <vespa/searchlib/memoryindex/url_field_inverter.h> +#include <vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h> #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/document/repo/fixedtyperepo.h> namespace search { @@ -183,7 +182,7 @@ struct Fixture DocBuilder _b; std::vector<std::unique_ptr<FieldInverter> > _inverters; std::unique_ptr<UrlFieldInverter> _urlInverter; - test::OrderedDocumentInserter _inserter; + test::OrderedFieldIndexInserter _inserter; index::SchemaIndexFields _schemaIndexFields; static Schema diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/.gitignore b/searchlib/src/tests/memoryindex/urlfieldinverter/.gitignore deleted file mode 100644 index b2636fe5e81..00000000000 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/.gitignore +++ /dev/null @@ -1 +0,0 @@ -searchlib_urlfieldinverter_test_app diff --git a/searchlib/src/tests/memoryindex/urlfieldinverter/CMakeLists.txt b/searchlib/src/tests/memoryindex/urlfieldinverter/CMakeLists.txt deleted file mode 100644 index 16fa8f5952e..00000000000 --- a/searchlib/src/tests/memoryindex/urlfieldinverter/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchlib_urlfieldinverter_test_app TEST - SOURCES - urlfieldinverter_test.cpp - DEPENDS - searchlib_test - searchlib -) -vespa_add_test(NAME searchlib_urlfieldinverter_test_app COMMAND searchlib_urlfieldinverter_test_app) diff --git a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt index 3619affb54e..b21b799e693 100644 --- a/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/diskindex/CMakeLists.txt @@ -18,6 +18,8 @@ vespa_add_library(searchlib_diskindex OBJECT pagedict4file.cpp pagedict4randread.cpp wordnummapper.cpp + zc4_posting_writer.cpp + zc4_posting_writer_base.cpp zcbuf.cpp zcposocc.cpp zcposocciterators.cpp diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp new file mode 100644 index 00000000000..0eb59a383a5 --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.cpp @@ -0,0 +1,270 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "zc4_posting_writer.h" +#include <vespa/searchlib/index/docidandfeatures.h> +#include <vespa/searchlib/index/postinglistcounts.h> + +using search::index::DocIdAndFeatures; +using search::index::PostingListCounts; +using search::index::PostingListParams; + +namespace search::diskindex +{ + +template <bool bigEndian> +Zc4PostingWriter<bigEndian>::Zc4PostingWriter(PostingListCounts &counts) + : Zc4PostingWriterBase(counts), + _encode_context(), + _encode_features(nullptr) +{ + _encode_context.setWriteContext(&_writeContext); + _writeContext.setEncodeContext(&_encode_context); +} + +template <bool bigEndian> +Zc4PostingWriter<bigEndian>::~Zc4PostingWriter() +{ +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::reset_chunk() +{ + _docIds.clear(); + if (_encode_features != nullptr) { + _encode_features->setupWrite(_featureWriteContext); + _featureOffset = 0; + } +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::flush_word_with_skip(bool hasMore) +{ + assert(_docIds.size() >= _minSkipDocs || !_counts._segments.empty()); + + if (_encode_features != nullptr) { + _encode_features->flush(); + } + EncodeContext &e = _encode_context; + + uint32_t numDocs = _docIds.size(); + + e.encodeExpGolomb(numDocs - 1, K_VALUE_ZCPOSTING_NUMDOCS); + if (numDocs >= _minChunkDocs) { + e.writeBits((hasMore ? 1 : 0), 1); + } + + calc_skip_info(_encode_features != nullptr); + + uint32_t docIdsSize = _zcDocIds.size(); + uint32_t l1SkipSize = _l1Skip.size(); + uint32_t l2SkipSize = _l2Skip.size(); + uint32_t l3SkipSize = _l3Skip.size(); + uint32_t l4SkipSize = _l4Skip.size(); + + e.encodeExpGolomb(docIdsSize - 1, K_VALUE_ZCPOSTING_DOCIDSSIZE); + e.encodeExpGolomb(l1SkipSize, K_VALUE_ZCPOSTING_L1SKIPSIZE); + if (l1SkipSize != 0) { + e.encodeExpGolomb(l2SkipSize, K_VALUE_ZCPOSTING_L2SKIPSIZE); + if (l2SkipSize != 0) { + e.encodeExpGolomb(l3SkipSize, K_VALUE_ZCPOSTING_L3SKIPSIZE); + if (l3SkipSize != 0) { + e.encodeExpGolomb(l4SkipSize, K_VALUE_ZCPOSTING_L4SKIPSIZE); + } + } + } + if (_encode_features != nullptr) { + e.encodeExpGolomb(_featureOffset, K_VALUE_ZCPOSTING_FEATURESSIZE); + } + + // Encode last document id in chunk or word. + if (_dynamicK) { + uint32_t docIdK = e.calcDocIdK((_counts._segments.empty() && + !hasMore) ? + numDocs : 1, + _docIdLimit); + e.encodeExpGolomb(_docIdLimit - 1 - _docIds.back().first, + docIdK); + } else { + e.encodeExpGolomb(_docIdLimit - 1 - _docIds.back().first, + K_VALUE_ZCPOSTING_LASTDOCID); + } + + e.smallAlign(8); // Byte align + + uint8_t *docIds = _zcDocIds._mallocStart; + e.writeBits(reinterpret_cast<const uint64_t *>(docIds), + 0, + docIdsSize * 8); + if (l1SkipSize > 0) { + uint8_t *l1Skip = _l1Skip._mallocStart; + e.writeBits(reinterpret_cast<const uint64_t *>(l1Skip), + 0, + l1SkipSize * 8); + } + if (l2SkipSize > 0) { + uint8_t *l2Skip = _l2Skip._mallocStart; + e.writeBits(reinterpret_cast<const uint64_t *>(l2Skip), + 0, + l2SkipSize * 8); + } + if (l3SkipSize > 0) { + uint8_t *l3Skip = _l3Skip._mallocStart; + e.writeBits(reinterpret_cast<const uint64_t *>(l3Skip), + 0, + l3SkipSize * 8); + } + if (l4SkipSize > 0) { + uint8_t *l4Skip = _l4Skip._mallocStart; + e.writeBits(reinterpret_cast<const uint64_t *>(l4Skip), + 0, + l4SkipSize * 8); + } + + // Write features + e.writeBits(static_cast<const uint64_t *>(_featureWriteContext._comprBuf), + 0, + _featureOffset); + + _counts._numDocs += numDocs; + if (hasMore || !_counts._segments.empty()) { + uint64_t writePos = e.getWriteOffset(); + PostingListCounts::Segment seg; + seg._bitLength = writePos - (_writePos + _counts._bitLength); + seg._numDocs = numDocs; + seg._lastDoc = _docIds.back().first; + _counts._segments.push_back(seg); + _counts._bitLength += seg._bitLength; + } + // reset tables in preparation for next word or next chunk + clear_skip_info(); + reset_chunk(); +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::write_docid_and_features(const DocIdAndFeatures &features) +{ + if (__builtin_expect(_docIds.size() >= _minChunkDocs, false)) { + flush_word_with_skip(true); + } + if (_encode_features != nullptr) { + _encode_features->writeFeatures(features); + uint64_t writeOffset = _encode_features->getWriteOffset(); + uint64_t featureSize = writeOffset - _featureOffset; + assert(static_cast<uint32_t>(featureSize) == featureSize); + _docIds.push_back(std::make_pair(features._docId, + static_cast<uint32_t>(featureSize))); + _featureOffset = writeOffset; + } else { + _docIds.push_back(std::make_pair(features._docId, uint32_t(0))); + } +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::flush_word_no_skip() +{ + // Too few document ids for skip info. + assert(_docIds.size() < _minSkipDocs && _counts._segments.empty()); + + if (_encode_features != nullptr) { + _encode_features->flush(); + } + EncodeContext &e = _encode_context; + uint32_t numDocs = _docIds.size(); + + e.encodeExpGolomb(numDocs - 1, K_VALUE_ZCPOSTING_NUMDOCS); + + uint32_t docIdK = _dynamicK ? e.calcDocIdK(numDocs, _docIdLimit) : K_VALUE_ZCPOSTING_DELTA_DOCID; + + uint32_t baseDocId = 1; + const uint64_t *features = + static_cast<const uint64_t *>(_featureWriteContext._comprBuf); + uint64_t featureOffset = 0; + + std::vector<DocIdAndFeatureSize>::const_iterator dit = _docIds.begin(); + std::vector<DocIdAndFeatureSize>::const_iterator dite = _docIds.end(); + + for (; dit != dite; ++dit) { + uint32_t docId = dit->first; + uint32_t featureSize = dit->second; + e.encodeExpGolomb(docId - baseDocId, docIdK); + baseDocId = docId + 1; + if (featureSize != 0) { + e.writeBits(features + (featureOffset >> 6), + featureOffset & 63, + featureSize); + featureOffset += featureSize; + } + } + _counts._numDocs += numDocs; + reset_chunk(); +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::flush_word() +{ + if (__builtin_expect(_docIds.size() >= _minSkipDocs || + !_counts._segments.empty(), false)) { + // Use skip information if enough documents or chunking has happened + flush_word_with_skip(false); + _numWords++; + } else if (_docIds.size() > 0) { + flush_word_no_skip(); + _numWords++; + } + + EncodeContext &e = _encode_context; + uint64_t writePos = e.getWriteOffset(); + + _counts._bitLength = writePos - _writePos; + _writePos = writePos; +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::set_encode_features(EncodeContext *encode_features) +{ + _encode_features = encode_features; + if (_encode_features != nullptr) { + _encode_features->setWriteContext(&_featureWriteContext); + _encode_features->setupWrite(_featureWriteContext); + } + _featureWriteContext.setEncodeContext(_encode_features); + _featureOffset = 0; +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::on_open() +{ + _numWords = 0; + _writePos = _encode_context.getWriteOffset(); // Position after file header +} + +template <bool bigEndian> +void +Zc4PostingWriter<bigEndian>::on_close() +{ + // Write some pad bits to avoid decompression readahead going past + // memory mapped file during search and into SIGSEGV territory. + + // First pad to 64 bits alignment. + _encode_context.smallAlign(64); + _encode_context.writeComprBufferIfNeeded(); + + // Then write 128 more bits. This allows for 64-bit decoding + // with a readbits that always leaves a nonzero preRead + _encode_context.padBits(128); + _encode_context.alignDirectIO(); + _encode_context.flush(); + _encode_context.writeComprBuffer(); // Also flushes slack +} + +template class Zc4PostingWriter<false>; +template class Zc4PostingWriter<true>; + +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.h b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.h new file mode 100644 index 00000000000..8dc5e249d52 --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer.h @@ -0,0 +1,53 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "zc4_posting_writer_base.h" + +namespace search::index { class DocIdAndFeatures; } + +namespace search::diskindex +{ + +/* + * Class used to write posting lists of type "Zc.4" and "Zc.5" (dynamic k). + * + * Common words have docid deltas and skip info separate from + * features. + * + * Rare words do not have skip info, and docid deltas and features are + * interleaved. + */ +template <bool bigEndian> +class Zc4PostingWriter : public Zc4PostingWriterBase +{ + using EncodeContext = bitcompression::FeatureEncodeContext<bigEndian>; + + EncodeContext _encode_context; + // Buffer up features in memory + EncodeContext *_encode_features; +public: + Zc4PostingWriter(const Zc4PostingWriter &) = delete; + Zc4PostingWriter(Zc4PostingWriter &&) = delete; + Zc4PostingWriter &operator=(const Zc4PostingWriter &) = delete; + Zc4PostingWriter &operator=(Zc4PostingWriter &&) = delete; + Zc4PostingWriter(index::PostingListCounts &counts); + ~Zc4PostingWriter(); + + void reset_chunk(); + void flush_word_with_skip(bool hasMore); + void flush_word_no_skip(); + void flush_word(); + void write_docid_and_features(const index::DocIdAndFeatures &features); + void set_encode_features(EncodeContext *encode_features); + void on_open(); + void on_close(); + + EncodeContext &get_encode_features() { return *_encode_features; } + EncodeContext &get_encode_context() { return _encode_context; } +}; + +extern template class Zc4PostingWriter<false>; +extern template class Zc4PostingWriter<true>; + +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp new file mode 100644 index 00000000000..485610c2ebd --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.cpp @@ -0,0 +1,222 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "zc4_posting_writer_base.h" +#include <vespa/searchlib/index/postinglistcounts.h> + +using search::index::PostingListCounts; +using search::index::PostingListParams; + +namespace search::diskindex +{ + +Zc4PostingWriterBase::Zc4PostingWriterBase(PostingListCounts &counts) + : _minChunkDocs(1 << 30), + _minSkipDocs(64), + _docIdLimit(10000000), + _docIds(), + _featureOffset(0), + _writePos(0), + _dynamicK(false), + _zcDocIds(), + _l1Skip(), + _l2Skip(), + _l3Skip(), + _l4Skip(), + _numWords(0), + _counts(counts), + _writeContext(sizeof(uint64_t)), + _featureWriteContext(sizeof(uint64_t)) +{ + _featureWriteContext.allocComprBuf(64, 1); + // Ensure that some space is initially available in encoding buffers + _zcDocIds.maybeExpand(); + _l1Skip.maybeExpand(); + _l2Skip.maybeExpand(); + _l3Skip.maybeExpand(); + _l4Skip.maybeExpand(); +} + +Zc4PostingWriterBase::~Zc4PostingWriterBase() +{ +} + +#define L1SKIPSTRIDE 16 +#define L2SKIPSTRIDE 8 +#define L3SKIPSTRIDE 8 +#define L4SKIPSTRIDE 8 + +void +Zc4PostingWriterBase::calc_skip_info(bool encodeFeatures) +{ + uint32_t lastDocId = 0u; + uint32_t lastL1SkipDocId = 0u; + uint32_t lastL1SkipDocIdPos = 0; + uint32_t lastL1SkipFeaturePos = 0; + uint32_t lastL2SkipDocId = 0u; + uint32_t lastL2SkipDocIdPos = 0; + uint32_t lastL2SkipFeaturePos = 0; + uint32_t lastL2SkipL1SkipPos = 0; + uint32_t lastL3SkipDocId = 0u; + uint32_t lastL3SkipDocIdPos = 0; + uint32_t lastL3SkipFeaturePos = 0; + uint32_t lastL3SkipL1SkipPos = 0; + uint32_t lastL3SkipL2SkipPos = 0; + uint32_t lastL4SkipDocId = 0u; + uint32_t lastL4SkipDocIdPos = 0; + uint32_t lastL4SkipFeaturePos = 0; + uint32_t lastL4SkipL1SkipPos = 0; + uint32_t lastL4SkipL2SkipPos = 0; + uint32_t lastL4SkipL3SkipPos = 0; + unsigned int l1SkipCnt = 0; + unsigned int l2SkipCnt = 0; + unsigned int l3SkipCnt = 0; + unsigned int l4SkipCnt = 0; + uint64_t featurePos = 0; + + std::vector<DocIdAndFeatureSize>::const_iterator dit = _docIds.begin(); + std::vector<DocIdAndFeatureSize>::const_iterator dite = _docIds.end(); + + if (!_counts._segments.empty()) { + lastDocId = _counts._segments.back()._lastDoc; + lastL1SkipDocId = lastDocId; + lastL2SkipDocId = lastDocId; + lastL3SkipDocId = lastDocId; + lastL4SkipDocId = lastDocId; + } + + for (; dit != dite; ++dit) { + if (l1SkipCnt >= L1SKIPSTRIDE) { + // L1 docid delta + uint32_t docIdDelta = lastDocId - lastL1SkipDocId; + assert(static_cast<int32_t>(docIdDelta) > 0); + _l1Skip.encode(docIdDelta - 1); + lastL1SkipDocId = lastDocId; + // L1 docid pos + uint64_t docIdPos = _zcDocIds.size(); + _l1Skip.encode(docIdPos - lastL1SkipDocIdPos - 1); + lastL1SkipDocIdPos = docIdPos; + if (encodeFeatures) { + // L1 features pos + _l1Skip.encode(featurePos - lastL1SkipFeaturePos - 1); + lastL1SkipFeaturePos = featurePos; + } + l1SkipCnt = 0; + ++l2SkipCnt; + if (l2SkipCnt >= L2SKIPSTRIDE) { + // L2 docid delta + docIdDelta = lastDocId - lastL2SkipDocId; + assert(static_cast<int32_t>(docIdDelta) > 0); + _l2Skip.encode(docIdDelta - 1); + lastL2SkipDocId = lastDocId; + // L2 docid pos + docIdPos = _zcDocIds.size(); + _l2Skip.encode(docIdPos - lastL2SkipDocIdPos - 1); + lastL2SkipDocIdPos = docIdPos; + if (encodeFeatures) { + // L2 features pos + _l2Skip.encode(featurePos - lastL2SkipFeaturePos - 1); + lastL2SkipFeaturePos = featurePos; + } + // L2 L1Skip pos + uint64_t l1SkipPos = _l1Skip.size(); + _l2Skip.encode(l1SkipPos - lastL2SkipL1SkipPos - 1); + lastL2SkipL1SkipPos = l1SkipPos; + l2SkipCnt = 0; + ++l3SkipCnt; + if (l3SkipCnt >= L3SKIPSTRIDE) { + // L3 docid delta + docIdDelta = lastDocId - lastL3SkipDocId; + assert(static_cast<int32_t>(docIdDelta) > 0); + _l3Skip.encode(docIdDelta - 1); + lastL3SkipDocId = lastDocId; + // L3 docid pos + docIdPos = _zcDocIds.size(); + _l3Skip.encode(docIdPos - lastL3SkipDocIdPos - 1); + lastL3SkipDocIdPos = docIdPos; + if (encodeFeatures) { + // L3 features pos + _l3Skip.encode(featurePos - lastL3SkipFeaturePos - 1); + lastL3SkipFeaturePos = featurePos; + } + // L3 L1Skip pos + l1SkipPos = _l1Skip.size(); + _l3Skip.encode(l1SkipPos - lastL3SkipL1SkipPos - 1); + lastL3SkipL1SkipPos = l1SkipPos; + // L3 L2Skip pos + uint64_t l2SkipPos = _l2Skip.size(); + _l3Skip.encode(l2SkipPos - lastL3SkipL2SkipPos - 1); + lastL3SkipL2SkipPos = l2SkipPos; + l3SkipCnt = 0; + ++l4SkipCnt; + if (l4SkipCnt >= L4SKIPSTRIDE) { + // L4 docid delta + docIdDelta = lastDocId - lastL4SkipDocId; + assert(static_cast<int32_t>(docIdDelta) > 0); + _l4Skip.encode(docIdDelta - 1); + lastL4SkipDocId = lastDocId; + // L4 docid pos + docIdPos = _zcDocIds.size(); + _l4Skip.encode(docIdPos - lastL4SkipDocIdPos - 1); + lastL4SkipDocIdPos = docIdPos; + if (encodeFeatures) { + // L4 features pos + _l4Skip.encode(featurePos - lastL4SkipFeaturePos - 1); + lastL4SkipFeaturePos = featurePos; + } + // L4 L1Skip pos + l1SkipPos = _l1Skip.size(); + _l4Skip.encode(l1SkipPos - lastL4SkipL1SkipPos - 1); + lastL4SkipL1SkipPos = l1SkipPos; + // L4 L2Skip pos + l2SkipPos = _l2Skip.size(); + _l4Skip.encode(l2SkipPos - lastL4SkipL2SkipPos - 1); + lastL4SkipL2SkipPos = l2SkipPos; + // L4 L3Skip pos + uint64_t l3SkipPos = _l3Skip.size(); + _l4Skip.encode(l3SkipPos - lastL4SkipL3SkipPos - 1); + lastL4SkipL3SkipPos = l3SkipPos; + l4SkipCnt = 0; + } + } + } + } + uint32_t docId = dit->first; + featurePos += dit->second; + _zcDocIds.encode(docId - lastDocId - 1); + lastDocId = docId; + ++l1SkipCnt; + } + // Extra partial entries for skip tables to simplify iterator during search + if (_l1Skip.size() > 0) { + _l1Skip.encode(lastDocId - lastL1SkipDocId - 1); + } + if (_l2Skip.size() > 0) { + _l2Skip.encode(lastDocId - lastL2SkipDocId - 1); + } + if (_l3Skip.size() > 0) { + _l3Skip.encode(lastDocId - lastL3SkipDocId - 1); + } + if (_l4Skip.size() > 0) { + _l4Skip.encode(lastDocId - lastL4SkipDocId - 1); + } +} + +void +Zc4PostingWriterBase::clear_skip_info() +{ + _zcDocIds.clear(); + _l1Skip.clear(); + _l2Skip.clear(); + _l3Skip.clear(); + _l4Skip.clear(); +} + +void +Zc4PostingWriterBase::set_posting_list_params(const PostingListParams ¶ms) +{ + params.get("docIdLimit", _docIdLimit); + params.get("minChunkDocs", _minChunkDocs); + params.get("minSkipDocs", _minSkipDocs); +} + +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h new file mode 100644 index 00000000000..ba781c11564 --- /dev/null +++ b/searchlib/src/vespa/searchlib/diskindex/zc4_posting_writer_base.h @@ -0,0 +1,66 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "zcbuf.h" +#include <vespa/searchlib/bitcompression/compression.h> +#include <vector> + +namespace search::index { +class PostingListCounts; +class PostingListParams; +} + +namespace search::diskindex +{ + +/* + * Base class for writing posting lists that might have basic skip info. + */ +class Zc4PostingWriterBase +{ +protected: + uint32_t _minChunkDocs; // # of documents needed for chunking + uint32_t _minSkipDocs; // # of documents needed for skipping + uint32_t _docIdLimit; // Limit for document ids (docId < docIdLimit) + + // Unpacked document ids for word and feature sizes + using DocIdAndFeatureSize = std::pair<uint32_t, uint32_t>; + std::vector<DocIdAndFeatureSize> _docIds; + + uint64_t _featureOffset; // Bit offset of next feature + uint64_t _writePos; // Bit position for start of current word + bool _dynamicK; // Caclulate EG compression parameters ? + ZcBuf _zcDocIds; // Document id deltas + ZcBuf _l1Skip; // L1 skip info + ZcBuf _l2Skip; // L2 skip info + ZcBuf _l3Skip; // L3 skip info + ZcBuf _l4Skip; // L4 skip info + + uint64_t _numWords; // Number of words in file + index::PostingListCounts &_counts; + search::ComprFileWriteContext _writeContext; + search::ComprFileWriteContext _featureWriteContext; + + Zc4PostingWriterBase(const Zc4PostingWriterBase &) = delete; + Zc4PostingWriterBase(Zc4PostingWriterBase &&) = delete; + Zc4PostingWriterBase &operator=(const Zc4PostingWriterBase &) = delete; + Zc4PostingWriterBase &operator=(Zc4PostingWriterBase &&) = delete; + Zc4PostingWriterBase(index::PostingListCounts &counts); + ~Zc4PostingWriterBase(); + void calc_skip_info(bool encodeFeatures); + void clear_skip_info(); + +public: + ComprFileWriteContext &get_write_context() { return _writeContext; } + ComprFileWriteContext &get_feature_write_context() { return _featureWriteContext; } + uint32_t get_min_chunk_docs() const { return _minChunkDocs; } + uint32_t get_min_skip_docs() const { return _minSkipDocs; } + uint32_t get_docid_limit() const { return _docIdLimit; } + uint64_t get_num_words() const { return _numWords; } + bool get_dynamic_k() const { return _dynamicK; } + void set_dynamic_k(bool dynamicK) { _dynamicK = dynamicK; } + void set_posting_list_params(const index::PostingListParams ¶ms); +}; + +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp index df06432816f..10c08af92cb 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocc.cpp @@ -63,9 +63,7 @@ Zc4PosOccSeqWrite::Zc4PosOccSeqWrite(const Schema &schema, _fieldsParams(), _realEncodeFeatures(&_fieldsParams) { - _encodeFeatures = &_realEncodeFeatures; - _encodeFeatures->setWriteContext(&_featureWriteContext); - _featureWriteContext.setEncodeContext(_encodeFeatures); + _writer.set_encode_features(&_realEncodeFeatures); _fieldsParams.setSchemaParams(schema, indexId); } @@ -118,9 +116,7 @@ ZcPosOccSeqWrite::ZcPosOccSeqWrite(const Schema &schema, _fieldsParams(), _realEncodeFeatures(&_fieldsParams) { - _encodeFeatures = &_realEncodeFeatures; - _encodeFeatures->setWriteContext(&_featureWriteContext); - _featureWriteContext.setEncodeContext(_encodeFeatures); + _writer.set_encode_features(&_realEncodeFeatures); _fieldsParams.setSchemaParams(schema, indexId); } diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp index d51a592bf2b..e850f169adc 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp @@ -607,36 +607,16 @@ Zc4PostingSeqRead::setPostingOffset(uint64_t offset, Zc4PostingSeqWrite:: Zc4PostingSeqWrite(PostingListCountFileSeqWrite *countFile) : PostingListFileSeqWrite(), - _encodeContext(), - _writeContext(_encodeContext), + _writer(_counts), _file(), - _minChunkDocs(1 << 30), - _minSkipDocs(64), - _docIdLimit(10000000), - _docIds(), - _encodeFeatures(nullptr), - _featureOffset(0), - _featureWriteContext(sizeof(uint64_t)), - _writePos(0), - _dynamicK(false), - _zcDocIds(), - _l1Skip(), - _l2Skip(), - _l3Skip(), - _l4Skip(), - _numWords(0), _fileBitSize(0), _countFile(countFile) { - _encodeContext.setWriteContext(&_writeContext); - if (_countFile != nullptr) { PostingListParams params; _countFile->getParams(params); - params.get("docIdLimit", _docIdLimit); - params.get("minChunkDocs", _minChunkDocs); + _writer.set_posting_list_params(params); } - _featureWriteContext.allocComprBuf(64, 1); } @@ -646,110 +626,27 @@ Zc4PostingSeqWrite::~Zc4PostingSeqWrite() void -Zc4PostingSeqWrite:: -writeDocIdAndFeatures(const DocIdAndFeatures &features) +Zc4PostingSeqWrite::writeDocIdAndFeatures(const DocIdAndFeatures &features) { - if (__builtin_expect(_docIds.size() >= _minChunkDocs, false)) - flushChunk(); - _encodeFeatures->writeFeatures(features); - uint64_t writeOffset = _encodeFeatures->getWriteOffset(); - uint64_t featureSize = writeOffset - _featureOffset; - assert(static_cast<uint32_t>(featureSize) == featureSize); - _docIds.push_back(std::make_pair(features._docId, - static_cast<uint32_t>(featureSize))); - _featureOffset = writeOffset; + _writer.write_docid_and_features(features); } void Zc4PostingSeqWrite::flushWord() { - if (__builtin_expect(_docIds.size() >= _minSkipDocs || - !_counts._segments.empty(), false)) { - // Use skip information if enough documents of chunking has happened - flushWordWithSkip(false); - _numWords++; - } else if (_docIds.size() > 0) { - flushWordNoSkip(); - _numWords++; - } - - EncodeContext &e = _encodeContext; - uint64_t writePos = e.getWriteOffset(); - - _counts._bitLength = writePos - _writePos; - _writePos = writePos; -} - - -uint32_t -Zc4PostingSeqWrite::readHeader(const vespalib::string &name) -{ - EncodeContext &f = *_encodeFeatures; - - FeatureDecodeContextBE d; - ComprFileReadContext drc(d); - FastOS_File file; - const vespalib::string &myId = _dynamicK ? myId5 : myId4; - - d.setReadContext(&drc); - bool res = file.OpenReadOnly(name.c_str()); - if (!res) { - LOG(error, "Could not open %s for reading file header: %s", - name.c_str(), getLastErrorString().c_str()); - LOG_ABORT("should not be reached"); - } - - drc.setFile(&file); - drc.setFileSize(file.GetSize()); - drc.allocComprBuf(512, 32768u); - d.emptyBuffer(0); - drc.readComprBuffer(); - - vespalib::FileHeader header; - d.readHeader(header, file.getSize()); - uint32_t headerLen = header.getSize(); - assert(header.hasTag("frozen")); - assert(header.hasTag("fileBitSize")); - assert(header.hasTag("format.0")); - assert(header.hasTag("format.1")); - assert(!header.hasTag("format.2")); - assert(header.hasTag("numWords")); - assert(header.hasTag("minChunkDocs")); - assert(header.hasTag("docIdLimit")); - assert(header.hasTag("minSkipDocs")); - assert(header.hasTag("endian")); - bool headerCompleted = header.getTag("frozen").asInteger() != 0; - uint64_t headerFileBitSize = header.getTag("fileBitSize").asInteger(); - headerLen += (-headerLen & 7); - assert(!headerCompleted || headerFileBitSize >= headerLen * 8); - (void) headerCompleted; - (void) headerFileBitSize; - assert(header.getTag("format.0").asString() == myId); - (void) myId; - assert(header.getTag("format.1").asString() == f.getIdentifier()); - _minChunkDocs = header.getTag("minChunkDocs").asInteger(); - _docIdLimit = header.getTag("docIdLimit").asInteger(); - _minSkipDocs = header.getTag("minSkipDocs").asInteger(); - assert(header.getTag("endian").asString() == "big"); - // Read feature decoding specific subheader using helper decode context - f.readHeader(header, "features."); - // Align on 64-bit unit - d.smallAlign(64); - assert(d.getReadOffset() == headerLen * 8); - file.Close(); - return headerLen; + _writer.flush_word(); } void Zc4PostingSeqWrite::makeHeader(const FileHeaderContext &fileHeaderContext) { - EncodeContext &f = *_encodeFeatures; - EncodeContext &e = _encodeContext; - ComprFileWriteContext &wce = _writeContext; + EncodeContext &f = _writer.get_encode_features(); + EncodeContext &e = _writer.get_encode_context(); + ComprFileWriteContext &wce = _writer.get_write_context(); - const vespalib::string &myId = _dynamicK ? myId5 : myId4; + const vespalib::string &myId = _writer.get_dynamic_k() ? myId5 : myId4; vespalib::FileHeader header; typedef vespalib::GenericHeader::Tag Tag; @@ -759,9 +656,9 @@ Zc4PostingSeqWrite::makeHeader(const FileHeaderContext &fileHeaderContext) header.putTag(Tag("format.0", myId)); header.putTag(Tag("format.1", f.getIdentifier())); header.putTag(Tag("numWords", 0)); - header.putTag(Tag("minChunkDocs", _minChunkDocs)); - header.putTag(Tag("docIdLimit", _docIdLimit)); - header.putTag(Tag("minSkipDocs", _minSkipDocs)); + header.putTag(Tag("minChunkDocs", _writer.get_min_chunk_docs())); + header.putTag(Tag("docIdLimit", _writer.get_docid_limit())); + header.putTag(Tag("minSkipDocs", _writer.get_min_skip_docs())); header.putTag(Tag("endian", "big")); header.putTag(Tag("desc", "Posting list file")); @@ -788,7 +685,7 @@ Zc4PostingSeqWrite::updateHeader() typedef vespalib::GenericHeader::Tag Tag; h.putTag(Tag("frozen", 1)); h.putTag(Tag("fileBitSize", _fileBitSize)); - h.putTag(Tag("numWords", _numWords)); + h.putTag(Tag("numWords", _writer.get_num_words())); h.rewriteFile(f); f.Sync(); f.Close(); @@ -813,40 +710,21 @@ Zc4PostingSeqWrite::open(const vespalib::string &name, // XXX may need to do something more here, I don't know what... return false; } - uint64_t fileSize = _file.GetSize(); - uint64_t bufferStartFilePos = _writeContext.getBufferStartFilePos(); - assert(fileSize >= bufferStartFilePos); - (void) fileSize; - _file.SetSize(bufferStartFilePos); - assert(bufferStartFilePos == static_cast<uint64_t>(_file.GetPosition())); - _writeContext.setFile(&_file); - search::ComprBuffer &cb = _writeContext; - EncodeContext &e = _encodeContext; - _writeContext.allocComprBuf(65536u, 32768u); - if (bufferStartFilePos == 0) { - e.setupWrite(cb); - // Reset accumulated stats - _fileBitSize = 0; - _numWords = 0; - // Start write initial header - makeHeader(fileHeaderContext); - _encodeFeatures->setupWrite(_featureWriteContext); - // end write initial header - _writePos = e.getWriteOffset(); - } else { - assert(bufferStartFilePos >= 8u); - uint32_t headerSize = readHeader(name); // Read existing header - assert(bufferStartFilePos >= headerSize); - (void) headerSize; - e.afterWrite(_writeContext, 0, bufferStartFilePos); - } - - // Ensure that some space is initially available in encoding buffers - _zcDocIds.maybeExpand(); - _l1Skip.maybeExpand(); - _l2Skip.maybeExpand(); - _l3Skip.maybeExpand(); - _l4Skip.maybeExpand(); + auto &writeContext = _writer.get_write_context(); + uint64_t bufferStartFilePos = writeContext.getBufferStartFilePos(); + assert(bufferStartFilePos == 0); + _file.SetSize(0); + writeContext.setFile(&_file); + search::ComprBuffer &cb = writeContext; + EncodeContext &e = _writer.get_encode_context(); + writeContext.allocComprBuf(65536u, 32768u); + e.setupWrite(cb); + // Reset accumulated stats + _fileBitSize = 0; + // Start write initial header + makeHeader(fileHeaderContext); + // end write initial header + _writer.on_open(); return true; // Assume success } @@ -854,42 +732,24 @@ Zc4PostingSeqWrite::open(const vespalib::string &name, bool Zc4PostingSeqWrite::close() { - EncodeContext &e = _encodeContext; - - _fileBitSize = e.getWriteOffset(); - // Write some pad bits to avoid decompression readahead going past - // memory mapped file during search and into SIGSEGV territory. - - // First pad to 64 bits alignment. - e.smallAlign(64); - e.writeComprBufferIfNeeded(); - - // Then write 128 more bits. This allows for 64-bit decoding - // with a readbits that always leaves a nonzero preRead - e.padBits(128); - e.alignDirectIO(); - e.flush(); - e.writeComprBuffer(); // Also flushes slack - - _writeContext.dropComprBuf(); + _fileBitSize = _writer.get_encode_context().getWriteOffset(); + _writer.on_close(); // flush and pad + auto &writeContext = _writer.get_write_context(); + writeContext.dropComprBuf(); _file.Sync(); _file.Close(); - _writeContext.setFile(nullptr); + writeContext.setFile(nullptr); updateHeader(); return true; } - - void Zc4PostingSeqWrite:: setParams(const PostingListParams ¶ms) { if (_countFile != nullptr) _countFile->setParams(params); - params.get("docIdLimit", _docIdLimit); - params.get("minChunkDocs", _minChunkDocs); - params.get("minSkipDocs", _minSkipDocs); + _writer.set_posting_list_params(params); } @@ -905,14 +765,14 @@ getParams(PostingListParams ¶ms) uint32_t countMinChunkDocs = 0; countParams.get("docIdLimit", countDocIdLimit); countParams.get("minChunkDocs", countMinChunkDocs); - assert(_docIdLimit == countDocIdLimit); - assert(_minChunkDocs == countMinChunkDocs); + assert(_writer.get_docid_limit() == countDocIdLimit); + assert(_writer.get_min_chunk_docs() == countMinChunkDocs); } else { params.clear(); - params.set("docIdLimit", _docIdLimit); - params.set("minChunkDocs", _minChunkDocs); + params.set("docIdLimit", _writer.get_docid_limit()); + params.set("minChunkDocs", _writer.get_min_chunk_docs()); } - params.set("minSkipDocs", _minSkipDocs); + params.set("minSkipDocs", _writer.get_min_skip_docs()); } @@ -920,7 +780,7 @@ void Zc4PostingSeqWrite:: setFeatureParams(const PostingListParams ¶ms) { - _encodeFeatures->setParams(params); + _writer.get_encode_features().setParams(params); } @@ -928,314 +788,7 @@ void Zc4PostingSeqWrite:: getFeatureParams(PostingListParams ¶ms) { - _encodeFeatures->getParams(params); -} - - -void -Zc4PostingSeqWrite::flushChunk() -{ - /* TODO: Flush chunk and prepare for new (possible short) chunk */ - flushWordWithSkip(true); -} - -#define L1SKIPSTRIDE 16 -#define L2SKIPSTRIDE 8 -#define L3SKIPSTRIDE 8 -#define L4SKIPSTRIDE 8 - - -void -Zc4PostingSeqWrite::calcSkipInfo() -{ - uint32_t lastDocId = 0u; - uint32_t lastL1SkipDocId = 0u; - uint32_t lastL1SkipDocIdPos = 0; - uint32_t lastL1SkipFeaturePos = 0; - uint32_t lastL2SkipDocId = 0u; - uint32_t lastL2SkipDocIdPos = 0; - uint32_t lastL2SkipFeaturePos = 0; - uint32_t lastL2SkipL1SkipPos = 0; - uint32_t lastL3SkipDocId = 0u; - uint32_t lastL3SkipDocIdPos = 0; - uint32_t lastL3SkipFeaturePos = 0; - uint32_t lastL3SkipL1SkipPos = 0; - uint32_t lastL3SkipL2SkipPos = 0; - uint32_t lastL4SkipDocId = 0u; - uint32_t lastL4SkipDocIdPos = 0; - uint32_t lastL4SkipFeaturePos = 0; - uint32_t lastL4SkipL1SkipPos = 0; - uint32_t lastL4SkipL2SkipPos = 0; - uint32_t lastL4SkipL3SkipPos = 0; - unsigned int l1SkipCnt = 0; - unsigned int l2SkipCnt = 0; - unsigned int l3SkipCnt = 0; - unsigned int l4SkipCnt = 0; - uint64_t featurePos = 0; - - std::vector<DocIdAndFeatureSize>::const_iterator dit = _docIds.begin(); - std::vector<DocIdAndFeatureSize>::const_iterator dite = _docIds.end(); - - if (!_counts._segments.empty()) { - lastDocId = _counts._segments.back()._lastDoc; - lastL1SkipDocId = lastDocId; - lastL2SkipDocId = lastDocId; - lastL3SkipDocId = lastDocId; - lastL4SkipDocId = lastDocId; - } - - for (; dit != dite; ++dit) { - if (l1SkipCnt >= L1SKIPSTRIDE) { - // L1 docid delta - uint32_t docIdDelta = lastDocId - lastL1SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - _l1Skip.encode(docIdDelta - 1); - lastL1SkipDocId = lastDocId; - // L1 docid pos - uint64_t docIdPos = _zcDocIds.size(); - _l1Skip.encode(docIdPos - lastL1SkipDocIdPos - 1); - lastL1SkipDocIdPos = docIdPos; - // L1 features pos - _l1Skip.encode(featurePos - lastL1SkipFeaturePos - 1); - lastL1SkipFeaturePos = featurePos; - l1SkipCnt = 0; - ++l2SkipCnt; - if (l2SkipCnt >= L2SKIPSTRIDE) { - // L2 docid delta - docIdDelta = lastDocId - lastL2SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - _l2Skip.encode(docIdDelta - 1); - lastL2SkipDocId = lastDocId; - // L2 docid pos - docIdPos = _zcDocIds.size(); - _l2Skip.encode(docIdPos - lastL2SkipDocIdPos - 1); - lastL2SkipDocIdPos = docIdPos; - // L2 features pos - _l2Skip.encode(featurePos - lastL2SkipFeaturePos - 1); - lastL2SkipFeaturePos = featurePos; - // L2 L1Skip pos - uint64_t l1SkipPos = _l1Skip.size(); - _l2Skip.encode(l1SkipPos - lastL2SkipL1SkipPos - 1); - lastL2SkipL1SkipPos = l1SkipPos; - l2SkipCnt = 0; - ++l3SkipCnt; - if (l3SkipCnt >= L3SKIPSTRIDE) { - // L3 docid delta - docIdDelta = lastDocId - lastL3SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - _l3Skip.encode(docIdDelta - 1); - lastL3SkipDocId = lastDocId; - // L3 docid pos - docIdPos = _zcDocIds.size(); - _l3Skip.encode(docIdPos - lastL3SkipDocIdPos - 1); - lastL3SkipDocIdPos = docIdPos; - // L3 features pos - _l3Skip.encode(featurePos - lastL3SkipFeaturePos - 1); - lastL3SkipFeaturePos = featurePos; - // L3 L1Skip pos - l1SkipPos = _l1Skip.size(); - _l3Skip.encode(l1SkipPos - lastL3SkipL1SkipPos - 1); - lastL3SkipL1SkipPos = l1SkipPos; - // L3 L2Skip pos - uint64_t l2SkipPos = _l2Skip.size(); - _l3Skip.encode(l2SkipPos - lastL3SkipL2SkipPos - 1); - lastL3SkipL2SkipPos = l2SkipPos; - l3SkipCnt = 0; - ++l4SkipCnt; - if (l4SkipCnt >= L4SKIPSTRIDE) { - // L4 docid delta - docIdDelta = lastDocId - lastL4SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - _l4Skip.encode(docIdDelta - 1); - lastL4SkipDocId = lastDocId; - // L4 docid pos - docIdPos = _zcDocIds.size(); - _l4Skip.encode(docIdPos - lastL4SkipDocIdPos - 1); - lastL4SkipDocIdPos = docIdPos; - // L4 features pos - _l4Skip.encode(featurePos - lastL4SkipFeaturePos - 1); - lastL4SkipFeaturePos = featurePos; - // L4 L1Skip pos - l1SkipPos = _l1Skip.size(); - _l4Skip.encode(l1SkipPos - lastL4SkipL1SkipPos - 1); - lastL4SkipL1SkipPos = l1SkipPos; - // L4 L2Skip pos - l2SkipPos = _l2Skip.size(); - _l4Skip.encode(l2SkipPos - lastL4SkipL2SkipPos - 1); - lastL4SkipL2SkipPos = l2SkipPos; - // L4 L3Skip pos - uint64_t l3SkipPos = _l3Skip.size(); - _l4Skip.encode(l3SkipPos - lastL4SkipL3SkipPos - 1); - lastL4SkipL3SkipPos = l3SkipPos; - l4SkipCnt = 0; - } - } - } - } - uint32_t docId = dit->first; - featurePos += dit->second; - _zcDocIds.encode(docId - lastDocId - 1); - lastDocId = docId; - ++l1SkipCnt; - } - // Extra partial entries for skip tables to simplify iterator during search - if (_l1Skip.size() > 0) - _l1Skip.encode(lastDocId - lastL1SkipDocId - 1); - if (_l2Skip.size() > 0) - _l2Skip.encode(lastDocId - lastL2SkipDocId - 1); - if (_l3Skip.size() > 0) - _l3Skip.encode(lastDocId - lastL3SkipDocId - 1); - if (_l4Skip.size() > 0) - _l4Skip.encode(lastDocId - lastL4SkipDocId - 1); -} - - -void -Zc4PostingSeqWrite::flushWordWithSkip(bool hasMore) -{ - assert(_docIds.size() >= _minSkipDocs || !_counts._segments.empty()); - - _encodeFeatures->flush(); - EncodeContext &e = _encodeContext; - - uint32_t numDocs = _docIds.size(); - - e.encodeExpGolomb(numDocs - 1, K_VALUE_ZCPOSTING_NUMDOCS); - if (numDocs >= _minChunkDocs) - e.writeBits((hasMore ? 1 : 0), 1); - - // TODO: Calculate docids size, possible also k parameter */ - calcSkipInfo(); - - uint32_t docIdsSize = _zcDocIds.size(); - uint32_t l1SkipSize = _l1Skip.size(); - uint32_t l2SkipSize = _l2Skip.size(); - uint32_t l3SkipSize = _l3Skip.size(); - uint32_t l4SkipSize = _l4Skip.size(); - - e.encodeExpGolomb(docIdsSize - 1, K_VALUE_ZCPOSTING_DOCIDSSIZE); - e.encodeExpGolomb(l1SkipSize, K_VALUE_ZCPOSTING_L1SKIPSIZE); - if (l1SkipSize != 0) { - e.encodeExpGolomb(l2SkipSize, K_VALUE_ZCPOSTING_L2SKIPSIZE); - if (l2SkipSize != 0) { - e.encodeExpGolomb(l3SkipSize, K_VALUE_ZCPOSTING_L3SKIPSIZE); - if (l3SkipSize != 0) { - e.encodeExpGolomb(l4SkipSize, K_VALUE_ZCPOSTING_L4SKIPSIZE); - } - } - } - e.encodeExpGolomb(_featureOffset, K_VALUE_ZCPOSTING_FEATURESSIZE); - - // Encode last document id in chunk or word. - if (_dynamicK) { - uint32_t docIdK = e.calcDocIdK((_counts._segments.empty() && - !hasMore) ? - numDocs : 1, - _docIdLimit); - e.encodeExpGolomb(_docIdLimit - 1 - _docIds.back().first, - docIdK); - } else { - e.encodeExpGolomb(_docIdLimit - 1 - _docIds.back().first, - K_VALUE_ZCPOSTING_LASTDOCID); - } - - e.smallAlign(8); // Byte align - - uint8_t *docIds = _zcDocIds._mallocStart; - e.writeBits(reinterpret_cast<const uint64_t *>(docIds), - 0, - docIdsSize * 8); - if (l1SkipSize > 0) { - uint8_t *l1Skip = _l1Skip._mallocStart; - e.writeBits(reinterpret_cast<const uint64_t *>(l1Skip), - 0, - l1SkipSize * 8); - } - if (l2SkipSize > 0) { - uint8_t *l2Skip = _l2Skip._mallocStart; - e.writeBits(reinterpret_cast<const uint64_t *>(l2Skip), - 0, - l2SkipSize * 8); - } - if (l3SkipSize > 0) { - uint8_t *l3Skip = _l3Skip._mallocStart; - e.writeBits(reinterpret_cast<const uint64_t *>(l3Skip), - 0, - l3SkipSize * 8); - } - if (l4SkipSize > 0) { - uint8_t *l4Skip = _l4Skip._mallocStart; - e.writeBits(reinterpret_cast<const uint64_t *>(l4Skip), - 0, - l4SkipSize * 8); - } - - // Write features - e.writeBits(static_cast<const uint64_t *>(_featureWriteContext._comprBuf), - 0, - _featureOffset); - - _counts._numDocs += numDocs; - if (hasMore || !_counts._segments.empty()) { - uint64_t writePos = e.getWriteOffset(); - PostingListCounts::Segment seg; - seg._bitLength = writePos - (_writePos + _counts._bitLength); - seg._numDocs = numDocs; - seg._lastDoc = _docIds.back().first; - _counts._segments.push_back(seg); - _counts._bitLength += seg._bitLength; - } - // reset tables in preparation for next word or next chunk - _zcDocIds.clear(); - _l1Skip.clear(); - _l2Skip.clear(); - _l3Skip.clear(); - _l4Skip.clear(); - resetWord(); -} - - -void -Zc4PostingSeqWrite::flushWordNoSkip() -{ - // Too few document ids for skip info. - assert(_docIds.size() < _minSkipDocs && _counts._segments.empty()); - - _encodeFeatures->flush(); - EncodeContext &e = _encodeContext; - uint32_t numDocs = _docIds.size(); - - e.encodeExpGolomb(numDocs - 1, K_VALUE_ZCPOSTING_NUMDOCS); - - uint32_t baseDocId = 1; - const uint64_t *features = - static_cast<const uint64_t *>(_featureWriteContext._comprBuf); - uint64_t featureOffset = 0; - - std::vector<DocIdAndFeatureSize>::const_iterator dit = _docIds.begin(); - std::vector<DocIdAndFeatureSize>::const_iterator dite = _docIds.end(); - - for (; dit != dite; ++dit) { - uint32_t docId = dit->first; - uint32_t featureSize = dit->second; - e.encodeExpGolomb(docId - baseDocId, K_VALUE_ZCPOSTING_DELTA_DOCID); - baseDocId = docId + 1; - e.writeBits(features + (featureOffset >> 6), - featureOffset & 63, - featureSize); - featureOffset += featureSize; - } - _counts._numDocs += numDocs; - resetWord(); -} - - -void -Zc4PostingSeqWrite::resetWord() -{ - _docIds.clear(); - _encodeFeatures->setupWrite(_featureWriteContext); - _featureOffset = 0; + _writer.get_encode_features().getParams(params); } @@ -1300,44 +853,7 @@ ZcPostingSeqRead::getIdentifier() ZcPostingSeqWrite::ZcPostingSeqWrite(PostingListCountFileSeqWrite *countFile) : Zc4PostingSeqWrite(countFile) { - _dynamicK = true; -} - - -void -ZcPostingSeqWrite::flushWordNoSkip() -{ - // Too few document ids for skip info. - assert(_docIds.size() < _minSkipDocs && _counts._segments.empty()); - - _encodeFeatures->flush(); - EncodeContext &e = _encodeContext; - uint32_t numDocs = _docIds.size(); - - e.encodeExpGolomb(numDocs - 1, K_VALUE_ZCPOSTING_NUMDOCS); - - uint32_t docIdK = e.calcDocIdK(numDocs, _docIdLimit); - - uint32_t baseDocId = 1; - const uint64_t *features = - static_cast<const uint64_t *>(_featureWriteContext._comprBuf); - uint64_t featureOffset = 0; - - std::vector<DocIdAndFeatureSize>::const_iterator dit = _docIds.begin(); - std::vector<DocIdAndFeatureSize>::const_iterator dite = _docIds.end(); - - for (; dit != dite; ++dit) { - uint32_t docId = dit->first; - uint32_t featureSize = dit->second; - e.encodeExpGolomb(docId - baseDocId, docIdK); - baseDocId = docId + 1; - e.writeBits(features + (featureOffset >> 6), - featureOffset & 63, - featureSize); - featureOffset += featureSize; - } - _counts._numDocs += numDocs; - resetWord(); + _writer.set_dynamic_k(true); } } // namespace search::diskindex diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.h b/searchlib/src/vespa/searchlib/diskindex/zcposting.h index 8c69a051e83..96cc306cea8 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.h @@ -2,9 +2,8 @@ #pragma once -#include "zcbuf.h" +#include "zc4_posting_writer.h" #include <vespa/searchlib/index/postinglistfile.h> -#include <vespa/searchlib/bitcompression/compression.h> #include <vespa/fastos/file.h> namespace search::index { @@ -131,29 +130,8 @@ class Zc4PostingSeqWrite : public index::PostingListFileSeqWrite protected: typedef bitcompression::FeatureEncodeContextBE EncodeContext; - EncodeContext _encodeContext; - search::ComprFileWriteContext _writeContext; - FastOS_File _file; - uint32_t _minChunkDocs; // # of documents needed for chunking - uint32_t _minSkipDocs; // # of documents needed for skipping - uint32_t _docIdLimit; // Limit for document ids (docId < docIdLimit) - // Unpacked document ids for word and feature sizes - typedef std::pair<uint32_t, uint32_t> DocIdAndFeatureSize; - std::vector<DocIdAndFeatureSize> _docIds; - - // Buffer up features in memory - EncodeContext *_encodeFeatures; - uint64_t _featureOffset; // Bit offset of next feature - search::ComprFileWriteContext _featureWriteContext; - uint64_t _writePos; // Bit position for start of current word - bool _dynamicK; // Caclulate EG compression parameters ? - ZcBuf _zcDocIds; // Document id deltas - ZcBuf _l1Skip; // L1 skip info - ZcBuf _l2Skip; // L2 skip info - ZcBuf _l3Skip; // L3 skip info - ZcBuf _l4Skip; // L4 skip info - - uint64_t _numWords; // Number of words in file + Zc4PostingWriter<true> _writer; + FastOS_File _file; uint64_t _fileBitSize; index::PostingListCountFileSeqWrite *const _countFile; public: @@ -177,37 +155,10 @@ public: void getFeatureParams(PostingListParams ¶ms) override; /** - * Flush chunk to file. - */ - void flushChunk(); - void calcSkipInfo(); - - /** - * Flush word with skip info to disk - */ - void flushWordWithSkip(bool hasMore); - - - /** - * Flush word without skip info to disk. - */ - virtual void flushWordNoSkip(); - - /** - * Prepare for next word or next chunk. - */ - void resetWord(); - - /** * Make header using feature encode write context. */ void makeHeader(const search::common::FileHeaderContext &fileHeaderContext); void updateHeader(); - - /** - * Read header, using temporary feature decode context. - */ - uint32_t readHeader(const vespalib::string &name); }; @@ -223,7 +174,6 @@ class ZcPostingSeqWrite : public Zc4PostingSeqWrite { public: ZcPostingSeqWrite(index::PostingListCountFileSeqWrite *countFile); - void flushWordNoSkip() override; }; } diff --git a/searchlib/src/vespa/searchlib/index/docidandfeatures.h b/searchlib/src/vespa/searchlib/index/docidandfeatures.h index 91a500495cc..a9329c9fa01 100644 --- a/searchlib/src/vespa/searchlib/index/docidandfeatures.h +++ b/searchlib/src/vespa/searchlib/index/docidandfeatures.h @@ -183,14 +183,6 @@ public: void setRaw(bool raw) { _raw = raw; } bool getRaw() const { return _raw; } - - /** - * Append features from a single field to a field collection. - * - * @param rhs features for a single field - * @param localFieldId local field id for the field - */ - void append(const DocIdAndFeatures &rhs, uint32_t localFieldId); }; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt index ffcd7ebd975..441fe12c383 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/memoryindex/CMakeLists.txt @@ -1,17 +1,17 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_memoryindex OBJECT SOURCES - compact_document_words_store.cpp - documentinverter.cpp - document_remover.cpp - featurestore.cpp + compact_words_store.cpp + document_inverter.cpp + feature_store.cpp field_index.cpp field_index_collection.cpp - fieldinverter.cpp - memoryindex.cpp - ordereddocumentinserter.cpp - postingiterator.cpp - urlfieldinverter.cpp - wordstore.cpp + field_index_remover.cpp + field_inverter.cpp + memory_index.cpp + ordered_field_index_inserter.cpp + posting_iterator.cpp + url_field_inverter.cpp + word_store.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp index e2d089626b1..27282282c11 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.cpp @@ -1,15 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "compact_document_words_store.h" +#include "compact_words_store.h" #include <vespa/searchlib/datastore/datastore.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/log/log.h> -LOG_SETUP(".memoryindex.compact_document_words_store"); +LOG_SETUP(".memoryindex.compact_words_store"); namespace search::memoryindex { -using Builder = CompactDocumentWordsStore::Builder; +using Builder = CompactWordsStore::Builder; namespace { @@ -36,28 +36,28 @@ serialize(const Builder &builder, uint32_t *begin) } -CompactDocumentWordsStore::Builder::Builder(uint32_t docId_) +CompactWordsStore::Builder::Builder(uint32_t docId_) : _docId(docId_), _words() { } -CompactDocumentWordsStore::Builder::~Builder() { } +CompactWordsStore::Builder::~Builder() { } -CompactDocumentWordsStore::Builder & -CompactDocumentWordsStore::Builder::insert(datastore::EntryRef wordRef) +CompactWordsStore::Builder & +CompactWordsStore::Builder::insert(datastore::EntryRef wordRef) { _words.push_back(wordRef); return *this; } inline void -CompactDocumentWordsStore::Iterator::nextWord() +CompactWordsStore::Iterator::nextWord() { _wordRef = *_buf++; _remainingWords--; } -CompactDocumentWordsStore::Iterator::Iterator() +CompactWordsStore::Iterator::Iterator() : _buf(nullptr), _remainingWords(0), _wordRef(0), @@ -65,7 +65,7 @@ CompactDocumentWordsStore::Iterator::Iterator() { } -CompactDocumentWordsStore::Iterator::Iterator(const uint32_t *buf) +CompactWordsStore::Iterator::Iterator(const uint32_t *buf) : _buf(buf), _remainingWords(0), _wordRef(0), @@ -79,8 +79,8 @@ CompactDocumentWordsStore::Iterator::Iterator(const uint32_t *buf) } } -CompactDocumentWordsStore::Iterator & -CompactDocumentWordsStore::Iterator::operator++() +CompactWordsStore::Iterator & +CompactWordsStore::Iterator::operator++() { if (_remainingWords > 0) { nextWord(); @@ -90,7 +90,7 @@ CompactDocumentWordsStore::Iterator::operator++() return *this; } -CompactDocumentWordsStore::Store::Store() +CompactWordsStore::Store::Store() : _store(), _type(1, MIN_BUFFER_ARRAYS, @@ -101,13 +101,13 @@ CompactDocumentWordsStore::Store::Store() _store.initActiveBuffers(); } -CompactDocumentWordsStore::Store::~Store() +CompactWordsStore::Store::~Store() { _store.dropBuffers(); } datastore::EntryRef -CompactDocumentWordsStore::Store::insert(const Builder &builder) +CompactWordsStore::Store::insert(const Builder &builder) { size_t serializedSize = getSerializedSize(builder); auto result = _store.rawAllocator<uint32_t>(_typeId).alloc(serializedSize); @@ -118,26 +118,26 @@ CompactDocumentWordsStore::Store::insert(const Builder &builder) return result.ref; } -CompactDocumentWordsStore::Iterator -CompactDocumentWordsStore::Store::get(datastore::EntryRef ref) const +CompactWordsStore::Iterator +CompactWordsStore::Store::get(datastore::EntryRef wordRef) const { - RefType internalRef(ref); + RefType internalRef(wordRef); const uint32_t *buf = _store.getEntry<uint32_t>(internalRef); return Iterator(buf); } -CompactDocumentWordsStore::CompactDocumentWordsStore() +CompactWordsStore::CompactWordsStore() : _docs(), _wordsStore() { } -CompactDocumentWordsStore::~CompactDocumentWordsStore() { } +CompactWordsStore::~CompactWordsStore() { } void -CompactDocumentWordsStore::insert(const Builder &builder) +CompactWordsStore::insert(const Builder &builder) { - datastore::EntryRef ref = _wordsStore.insert(builder); - auto insres = _docs.insert(std::make_pair(builder.docId(), ref)); + datastore::EntryRef wordRef = _wordsStore.insert(builder); + auto insres = _docs.insert(std::make_pair(builder.docId(), wordRef)); if (!insres.second) { LOG(error, "Failed inserting remove info for docid %u", builder.docId()); @@ -146,13 +146,13 @@ CompactDocumentWordsStore::insert(const Builder &builder) } void -CompactDocumentWordsStore::remove(uint32_t docId) +CompactWordsStore::remove(uint32_t docId) { _docs.erase(docId); } -CompactDocumentWordsStore::Iterator -CompactDocumentWordsStore::get(uint32_t docId) const +CompactWordsStore::Iterator +CompactWordsStore::get(uint32_t docId) const { auto itr = _docs.find(docId); if (itr != _docs.end()) { @@ -162,7 +162,7 @@ CompactDocumentWordsStore::get(uint32_t docId) const } MemoryUsage -CompactDocumentWordsStore::getMemoryUsage() const +CompactWordsStore::getMemoryUsage() const { MemoryUsage usage; usage.incAllocatedBytes(_docs.getMemoryConsumption()); diff --git a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.h b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h index ced7ec241bd..2fc6ec8d5bb 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/compact_document_words_store.h +++ b/searchlib/src/vespa/searchlib/memoryindex/compact_words_store.h @@ -10,18 +10,16 @@ namespace search::memoryindex { /** - * Class used to store the {wordRef, fieldId, docId} tuples that are inserted - * into the memory index dictionary. These tuples are later used when removing - * all remains of a document from the posting lists of the dictionary. + * Class used to store the {wordRef, docId} tuples that are inserted into a FieldIndex and its posting lists. + * + * These tuples are later used when removing all remains of a document from the posting lists in that index. */ -class CompactDocumentWordsStore -{ +class CompactWordsStore { public: /** - * Builder used to collect all wordRefs for a field. + * Builder used to collect all words (as wordRefs) for a docId in a field. */ - class Builder - { + class Builder { public: using UP = std::unique_ptr<Builder>; using WordRefVector = vespalib::Array<datastore::EntryRef>; @@ -39,10 +37,9 @@ public: }; /** - * Iterator over all {wordRef, fieldId} pairs for a document. + * Iterator over all words (as wordRefs) for a docId in a field. */ - class Iterator - { + class Iterator { private: const uint32_t *_buf; uint32_t _remainingWords; @@ -61,10 +58,9 @@ public: }; /** - * Store for all {wordRef, fieldId} pairs among all documents. + * Store for all unique words (as wordRefs) among all documents. */ - class Store - { + class Store { public: using DataStoreType = datastore::DataStoreT<datastore::EntryRefT<22>>; using RefType = DataStoreType::RefType; @@ -78,7 +74,7 @@ public: Store(); ~Store(); datastore::EntryRef insert(const Builder &builder); - Iterator get(datastore::EntryRef ref) const; + Iterator get(datastore::EntryRef wordRef) const; MemoryUsage getMemoryUsage() const { return _store.getMemoryUsage(); } }; @@ -89,8 +85,8 @@ private: Store _wordsStore; public: - CompactDocumentWordsStore(); - ~CompactDocumentWordsStore(); + CompactWordsStore(); + ~CompactWordsStore(); void insert(const Builder &builder); void remove(uint32_t docId); Iterator get(uint32_t docId) const; diff --git a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp index 1501ff7d2fc..a468428e21f 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.cpp @@ -1,10 +1,10 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "documentinverter.h" +#include "document_inverter.h" #include "field_index_collection.h" -#include "fieldinverter.h" -#include "ordereddocumentinserter.h" -#include "urlfieldinverter.h" +#include "field_inverter.h" +#include "ordered_field_index_inserter.h" +#include "url_field_inverter.h" #include <vespa/document/annotation/alternatespanlist.h> #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/repo/fixedtyperepo.h> @@ -16,7 +16,7 @@ #include <stdexcept> #include <vespa/log/log.h> -LOG_SETUP(".memoryindex.documentinverter"); +LOG_SETUP(".memoryindex.document_inverter"); namespace search::memoryindex { @@ -39,7 +39,6 @@ using index::DocIdAndPosOccFeatures; using index::Schema; using search::util::URL; - DocumentInverter::DocumentInverter(const Schema &schema, ISequencedTaskExecutor &invertThreads, ISequencedTaskExecutor &pushThreads) @@ -74,14 +73,12 @@ DocumentInverter::DocumentInverter(const Schema &schema, } } - DocumentInverter::~DocumentInverter() { _invertThreads.sync(); _pushThreads.sync(); } - void DocumentInverter::addFieldPath(const document::DocumentType &docType, uint32_t fieldId) @@ -100,9 +97,9 @@ DocumentInverter::addFieldPath(const document::DocumentType &docType, _indexedFieldPaths[fieldId] = std::move(fp); } - -void DocumentInverter::buildFieldPath(const document::DocumentType &docType, - const document::DataType *dataType) +void +DocumentInverter::buildFieldPath(const document::DocumentType &docType, + const document::DataType *dataType) { _indexedFieldPaths.clear(); _indexedFieldPaths.resize(_schema.getNumIndexFields()); @@ -115,7 +112,6 @@ void DocumentInverter::buildFieldPath(const document::DocumentType &docType, _dataType = dataType; } - void DocumentInverter::invertDocument(uint32_t docId, const Document &doc) { @@ -154,7 +150,6 @@ DocumentInverter::invertDocument(uint32_t docId, const Document &doc) } } - void DocumentInverter::removeDocument(uint32_t docId) { @@ -175,7 +170,6 @@ DocumentInverter::removeDocument(uint32_t docId) } } - void DocumentInverter::pushDocuments(FieldIndexCollection &fieldIndexes, const std::shared_ptr<IDestructorCallback> &onWriteDone) @@ -184,8 +178,8 @@ DocumentInverter::pushDocuments(FieldIndexCollection &fieldIndexes, uint32_t fieldId = 0; for (auto &inverter : _inverters) { FieldIndex &fieldIndex(**indexFieldIterator); - DocumentRemover &remover(fieldIndex.getDocumentRemover()); - OrderedDocumentInserter &inserter(fieldIndex.getInserter()); + FieldIndexRemover &remover(fieldIndex.getDocumentRemover()); + OrderedFieldIndexInserter &inserter(fieldIndex.getInserter()); _pushThreads.execute(fieldId, [inverter(inverter.get()), &remover, &inserter, &fieldIndex, onWriteDone]() diff --git a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.h b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h index fa8d13d98fc..5c2d9cc84ed 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/documentinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter.h @@ -2,7 +2,7 @@ #pragma once -#include "i_document_remove_listener.h" +#include "i_field_index_remove_listener.h" #include <vespa/searchlib/index/schema_index_fields.h> namespace document { @@ -24,8 +24,7 @@ class FieldInverter; class UrlFieldInverter; class FieldIndexCollection; -class DocumentInverter -{ +class DocumentInverter { private: DocumentInverter(const DocumentInverter &) = delete; DocumentInverter &operator=(const DocumentInverter &) = delete; diff --git a/searchlib/src/vespa/searchlib/memoryindex/featurestore.cpp b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp index c032bb33217..974fcc01c36 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/featurestore.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "featurestore.h" +#include "feature_store.h" #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/datastore/datastore.hpp> @@ -26,7 +26,6 @@ FeatureStore::writeFeatures(uint32_t packedIndex, const DocIdAndFeatures &featur return oldOffset; } - datastore::EntryRef FeatureStore::addFeatures(const uint8_t *src, uint64_t byteLen) { @@ -43,7 +42,6 @@ FeatureStore::addFeatures(const uint8_t *src, uint64_t byteLen) return result.ref; } - std::pair<datastore::EntryRef, uint64_t> FeatureStore::addFeatures(uint64_t beginOffset, uint64_t endOffset) { @@ -58,7 +56,6 @@ FeatureStore::addFeatures(uint64_t beginOffset, uint64_t endOffset) return std::make_pair(ref, bitLen); } - datastore::EntryRef FeatureStore::moveFeatures(datastore::EntryRef ref, uint64_t bitLen) { @@ -70,7 +67,6 @@ FeatureStore::moveFeatures(datastore::EntryRef ref, uint64_t bitLen) return newRef; } - FeatureStore::FeatureStore(const Schema &schema) : _store(), _f(nullptr), @@ -95,13 +91,11 @@ FeatureStore::FeatureStore(const Schema &schema) _store.initActiveBuffers(); } - FeatureStore::~FeatureStore() { _store.dropBuffers(); } - std::pair<datastore::EntryRef, uint64_t> FeatureStore::addFeatures(uint32_t packedIndex, const DocIdAndFeatures &features) { @@ -111,8 +105,6 @@ FeatureStore::addFeatures(uint32_t packedIndex, const DocIdAndFeatures &features return addFeatures(oldOffset, newOffset); } - - void FeatureStore::getFeatures(uint32_t packedIndex, datastore::EntryRef ref, DocIdAndFeatures &features) { @@ -121,7 +113,6 @@ FeatureStore::getFeatures(uint32_t packedIndex, datastore::EntryRef ref, DocIdAn _d.readFeatures(features); } - size_t FeatureStore::bitSize(uint32_t packedIndex, datastore::EntryRef ref) { @@ -135,7 +126,6 @@ FeatureStore::bitSize(uint32_t packedIndex, datastore::EntryRef ref) return bitLen; } - datastore::EntryRef FeatureStore::moveFeatures(uint32_t packedIndex, datastore::EntryRef ref) { diff --git a/searchlib/src/vespa/searchlib/memoryindex/featurestore.h b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h index ef75b9f6d31..94d44eaf44d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/featurestore.h +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h @@ -9,8 +9,7 @@ namespace search::memoryindex { -class FeatureStore -{ +class FeatureStore { public: using DataStoreType = datastore::DataStoreT<datastore::AlignedEntryRefT<22, 2>>; using RefType = DataStoreType::RefType; @@ -122,9 +121,7 @@ public: * @param packedIndex The field or field collection owning features * @param decoder The feature decoder */ - void - setupForField(uint32_t packedIndex, DecodeContextCooked &decoder) const - { + void setupForField(uint32_t packedIndex, DecodeContextCooked &decoder) const { decoder._fieldsParams = &_fieldsParams[packedIndex]; } @@ -135,9 +132,7 @@ public: * @param ref Reference to stored features * @param decoder The feature decoder */ - void - setupForReadFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const - { + void setupForReadFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const { const uint8_t * bits = getBits(ref); decoder.setByteCompr(bits); uint32_t bufferId = RefType(ref).bufferId(); @@ -155,9 +150,7 @@ public: * @param ref Reference to stored features * @param decoder The feature decoder */ - void - setupForUnpackFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const - { + void setupForUnpackFeatures(datastore::EntryRef ref, DecodeContextCooked &decoder) const { decoder.setByteCompr(getBits(ref)); } @@ -169,8 +162,7 @@ public: * @param ref Reference to stored features * @return size of features in bits */ - size_t - bitSize(uint32_t packedIndex, datastore::EntryRef ref); + size_t bitSize(uint32_t packedIndex, datastore::EntryRef ref); /** * Get byte address of stored features diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index 4d42b9ae493..7d10895c32f 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "field_index.h" -#include "ordereddocumentinserter.h" +#include "ordered_field_index_inserter.h" #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/searchlib/bitcompression/posocccompression.h> @@ -38,7 +38,7 @@ FieldIndex::FieldIndex(const Schema & schema, uint32_t fieldId) _featureStore(schema), _fieldId(fieldId), _remover(_wordStore), - _inserter(std::make_unique<OrderedDocumentInserter>(*this)) + _inserter(std::make_unique<OrderedFieldIndexInserter>(*this)) { } FieldIndex::~FieldIndex() @@ -88,7 +88,6 @@ FieldIndex::findFrozen(const vespalib::stringref word) const return PostingList::Iterator(); } - void FieldIndex::compactFeatures() { @@ -218,7 +217,6 @@ FieldIndex::dump(search::index::IndexBuilder & indexBuilder) } } - MemoryUsage FieldIndex::getMemoryUsage() const { @@ -231,7 +229,7 @@ FieldIndex::getMemoryUsage() const return usage; } -} // namespace search::memoryindex +} namespace search::btree { diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.h b/searchlib/src/vespa/searchlib/memoryindex/field_index.h index 4a27e30b47a..3b0675b5fdf 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.h @@ -2,12 +2,12 @@ #pragma once -#include "featurestore.h" -#include "wordstore.h" -#include "document_remover.h" -#include <vespa/searchlib/btree/btreeroot.h> +#include "feature_store.h" +#include "field_index_remover.h" +#include "word_store.h" #include <vespa/searchlib/btree/btree.h> #include <vespa/searchlib/btree/btreenodeallocator.h> +#include <vespa/searchlib/btree/btreeroot.h> #include <vespa/searchlib/btree/btreestore.h> #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/searchlib/index/indexbuilder.h> @@ -16,7 +16,7 @@ namespace search::memoryindex { -class OrderedDocumentInserter; +class OrderedFieldIndexInserter; /** * Memory index for a single field. @@ -56,9 +56,7 @@ public: const WordStore &_wordStore; const vespalib::stringref _word; - const char * - getWord(datastore::EntryRef wordRef) const - { + const char *getWord(datastore::EntryRef wordRef) const { if (wordRef.valid()) { return _wordStore.getWord(wordRef); } @@ -71,9 +69,7 @@ public: _word(word) { } - bool - operator()(const WordKey & lhs, const WordKey & rhs) const - { + bool operator()(const WordKey & lhs, const WordKey & rhs) const { int cmpres = strcmp(getWord(lhs._wordRef), getWord(rhs._wordRef)); return cmpres < 0; } @@ -93,8 +89,8 @@ private: PostingListStore _postingListStore; FeatureStore _featureStore; uint32_t _fieldId; - DocumentRemover _remover; - std::unique_ptr<OrderedDocumentInserter> _inserter; + FieldIndexRemover _remover; + std::unique_ptr<OrderedFieldIndexInserter> _inserter; public: datastore::EntryRef addWord(const vespalib::stringref word) { @@ -102,9 +98,7 @@ public: return _wordStore.addWord(word); } - datastore::EntryRef - addFeatures(const index::DocIdAndFeatures &features) - { + datastore::EntryRef addFeatures(const index::DocIdAndFeatures &features) { return _featureStore.addFeatures(_fieldId, features).first; } @@ -118,7 +112,7 @@ public: uint64_t getNumUniqueWords() const { return _numUniqueWords; } const FeatureStore & getFeatureStore() const { return _featureStore; } const WordStore &getWordStore() const { return _wordStore; } - OrderedDocumentInserter &getInserter() const { return *_inserter; } + OrderedFieldIndexInserter &getInserter() const { return *_inserter; } private: void freeze() { @@ -126,9 +120,7 @@ private: _dict.getAllocator().freeze(); } - void - trimHoldLists() - { + void trimHoldLists() { GenerationHandler::generation_t usedGen = _generationHandler.getFirstUsedGeneration(); _postingListStore.trimHoldLists(usedGen); @@ -136,9 +128,7 @@ private: _featureStore.trimHoldLists(usedGen); } - void - transferHoldLists() - { + void transferHoldLists() { GenerationHandler::generation_t generation = _generationHandler.getCurrentGeneration(); _postingListStore.transferHoldLists(generation); @@ -146,9 +136,7 @@ private: _featureStore.transferHoldLists(generation); } - void - incGeneration() - { + void incGeneration() { _generationHandler.incGeneration(); } @@ -163,27 +151,11 @@ public: void dump(search::index::IndexBuilder & indexBuilder); MemoryUsage getMemoryUsage() const; + DictionaryTree &getDictionaryTree() { return _dict; } + PostingListStore &getPostingListStore() { return _postingListStore; } + FieldIndexRemover &getDocumentRemover() { return _remover; } - DictionaryTree & - getDictionaryTree() - { - return _dict; - } - - PostingListStore & - getPostingListStore() - { - return _postingListStore; - } - - DocumentRemover & - getDocumentRemover() - { - return _remover; - } - - void commit() - { + void commit() { _remover.flush(); freeze(); transferHoldLists(); diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp index 45431f0e8ef..27944b5fe89 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.cpp @@ -1,7 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "field_index_collection.h" -#include "fieldinverter.h" +#include "field_inverter.h" #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/btree/btreenode.hpp> @@ -40,7 +40,6 @@ FieldIndexCollection::~FieldIndexCollection() { } - void FieldIndexCollection::dump(search::index::IndexBuilder &indexBuilder) { @@ -61,6 +60,5 @@ FieldIndexCollection::getMemoryUsage() const return usage; } - } } diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h index 3b8e63626bf..5c2aa6f9b2c 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_collection.h @@ -6,7 +6,7 @@ namespace search::memoryindex { -class IDocumentRemoveListener; +class IFieldIndexRemoveListener; class FieldInverter; /** @@ -29,14 +29,11 @@ public: FieldIndexCollection(const index::Schema &schema); ~FieldIndexCollection(); PostingList::Iterator find(const vespalib::stringref word, - uint32_t fieldId) const - { + uint32_t fieldId) const { return _fieldIndexes[fieldId]->find(word); } - PostingList::ConstIterator - findFrozen(const vespalib::stringref word, uint32_t fieldId) const - { + PostingList::ConstIterator findFrozen(const vespalib::stringref word, uint32_t fieldId) const { return _fieldIndexes[fieldId]->findFrozen(word); } @@ -56,8 +53,7 @@ public: return _fieldIndexes[fieldId].get(); } - const std::vector<std::unique_ptr<FieldIndex>> & - getFieldIndexes() const { return _fieldIndexes; } + const std::vector<std::unique_ptr<FieldIndex>> &getFieldIndexes() const { return _fieldIndexes; } uint32_t getNumFields() const { return _numFields; } }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_remover.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp index 67b519bbadc..2afddf072f2 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_remover.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.cpp @@ -1,15 +1,16 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "document_remover.h" -#include "i_document_remove_listener.h" -#include "wordstore.h" + +#include "field_index_remover.h" +#include "i_field_index_remove_listener.h" +#include "word_store.h" #include <vespa/searchlib/common/sort.h> namespace search::memoryindex { -using Builder = CompactDocumentWordsStore::Builder; -using Iterator = CompactDocumentWordsStore::Iterator; +using Builder = CompactWordsStore::Builder; +using Iterator = CompactWordsStore::Iterator; -DocumentRemover::DocumentRemover(const WordStore &wordStore) +FieldIndexRemover::FieldIndexRemover(const WordStore &wordStore) : _store(), _builder(), _wordFieldDocTuples(), @@ -17,11 +18,10 @@ DocumentRemover::DocumentRemover(const WordStore &wordStore) { } -DocumentRemover::~DocumentRemover() { -} +FieldIndexRemover::~FieldIndexRemover() = default; void -DocumentRemover::remove(uint32_t docId, IDocumentRemoveListener &listener) +FieldIndexRemover::remove(uint32_t docId, IFieldIndexRemoveListener &listener) { Iterator itr = _store.get(docId); if (itr.valid()) { @@ -34,14 +34,13 @@ DocumentRemover::remove(uint32_t docId, IDocumentRemoveListener &listener) } void -DocumentRemover::insert(datastore::EntryRef wordRef, uint32_t docId) +FieldIndexRemover::insert(datastore::EntryRef wordRef, uint32_t docId) { _wordFieldDocTuples.emplace_back(wordRef, docId); } - void -DocumentRemover::flush() +FieldIndexRemover::flush() { if (_wordFieldDocTuples.empty()) { return; @@ -60,5 +59,4 @@ DocumentRemover::flush() _wordFieldDocTuples.clear(); } - } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_remover.h b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h index 5d44a666ff9..19b3353a27a 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_remover.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index_remover.h @@ -1,22 +1,24 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "compact_document_words_store.h" -#include "i_document_insert_listener.h" +#include "compact_words_store.h" +#include "i_field_index_insert_listener.h" namespace search::memoryindex { -class IDocumentRemoveListener; +class IFieldIndexRemoveListener; class WordStore; /** - * Class used to remove documents from the memory index dictionary. + * Class used to handle removal of documents from a FieldIndex. + * + * It tracks all {word, docId} tuples that are inserted into the index, + * and when removing a document, all these {word, docId} tuples are sent to the component + * that is doing the actual removal (IFieldIndexRemoveListener). */ -class DocumentRemover : public IDocumentInsertListener -{ +class FieldIndexRemover : public IFieldIndexInsertListener { private: - struct WordFieldDocTuple - { + struct WordFieldDocTuple { datastore::EntryRef _wordRef; uint32_t _docId; WordFieldDocTuple() : @@ -38,22 +40,21 @@ private: return wft._docId; } }; - }; - CompactDocumentWordsStore _store; - CompactDocumentWordsStore::Builder::UP _builder; + CompactWordsStore _store; + CompactWordsStore::Builder::UP _builder; std::vector<WordFieldDocTuple> _wordFieldDocTuples; const WordStore &_wordStore; public: - DocumentRemover(const WordStore &wordStore); - ~DocumentRemover(); - void remove(uint32_t docId, IDocumentRemoveListener &inverter); - CompactDocumentWordsStore &getStore() { return _store; } - const CompactDocumentWordsStore &getStore() const { return _store; } + FieldIndexRemover(const WordStore &wordStore); + ~FieldIndexRemover(); + void remove(uint32_t docId, IFieldIndexRemoveListener &inverter); + CompactWordsStore &getStore() { return _store; } + const CompactWordsStore &getStore() const { return _store; } - // Implements IDocumentInsertListener + // Implements IFieldIndexInsertListener void insert(datastore::EntryRef wordRef, uint32_t docId) override; void flush() override; }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp index fa261a4e90a..d19f05a98ee 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.cpp @@ -1,25 +1,25 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "fieldinverter.h" -#include "ordereddocumentinserter.h" +#include "field_inverter.h" +#include "ordered_field_index_inserter.h" +#include <vespa/document/annotation/alternatespanlist.h> +#include <vespa/document/annotation/annotation.h> +#include <vespa/document/annotation/span.h> +#include <vespa/document/annotation/spanlist.h> +#include <vespa/document/annotation/spantree.h> +#include <vespa/document/annotation/spantreevisitor.h> #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> +#include <vespa/searchlib/bitcompression/compression.h> +#include <vespa/searchlib/bitcompression/posocccompression.h> +#include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/util/url.h> -#include <stdexcept> -#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> +#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/sort.h> -#include <vespa/searchlib/bitcompression/compression.h> -#include <vespa/searchlib/bitcompression/posocccompression.h> -#include <vespa/document/annotation/annotation.h> -#include <vespa/document/annotation/span.h> -#include <vespa/document/annotation/spanlist.h> -#include <vespa/document/annotation/alternatespanlist.h> -#include <vespa/document/annotation/spantree.h> -#include <vespa/document/annotation/spantreevisitor.h> +#include <stdexcept> namespace search::memoryindex { @@ -48,23 +48,17 @@ using search::index::schema::CollectionType; using search::util::URL; using vespalib::make_string; -namespace documentinverterkludge { - -namespace linguistics { +namespace documentinverterkludge::linguistics { const vespalib::string SPANTREE_NAME("linguistics"); } -} - using namespace documentinverterkludge; -namespace -{ +namespace { -class SpanFinder : public SpanTreeVisitor -{ +class SpanFinder : public SpanTreeVisitor { public: int32_t begin_pos; int32_t end_pos; @@ -165,7 +159,6 @@ FieldInverter::processAnnotations(const StringFieldValue &value) } } - void FieldInverter::reset() { @@ -228,14 +221,12 @@ FieldInverter::sortWords() } } - void FieldInverter::startElement(int32_t weight) { _elems.push_back(ElemInfo(weight)); // Fill in length later } - void FieldInverter::endElement() { @@ -270,7 +261,6 @@ FieldInverter::saveWord(const vespalib::stringref word) return wordRef; } - uint32_t FieldInverter::saveWord(const document::FieldValue &fv) { @@ -280,7 +270,6 @@ FieldInverter::saveWord(const document::FieldValue &fv) return saveWord(vespalib::stringref(sRef.first, sRef.second)); } - void FieldInverter::remove(const vespalib::stringref word, uint32_t docId) { @@ -289,7 +278,6 @@ FieldInverter::remove(const vespalib::stringref word, uint32_t docId) _positions.emplace_back(wordRef, docId); } - void FieldInverter::processNormalDocTextField(const StringFieldValue &field) { @@ -298,7 +286,6 @@ FieldInverter::processNormalDocTextField(const StringFieldValue &field) endElement(); } - void FieldInverter::processNormalDocArrayTextField(const ArrayFieldValue &field) { @@ -314,7 +301,6 @@ FieldInverter::processNormalDocArrayTextField(const ArrayFieldValue &field) } } - void FieldInverter::processNormalDocWeightedSetTextField(const WeightedSetFieldValue &field) { @@ -331,7 +317,6 @@ FieldInverter::processNormalDocWeightedSetTextField(const WeightedSetFieldValue } } - FieldInverter::FieldInverter(const Schema &schema, uint32_t fieldId) : _fieldId(fieldId), _elem(0u), @@ -352,7 +337,6 @@ FieldInverter::FieldInverter(const Schema &schema, uint32_t fieldId) { } - void FieldInverter::abortPendingDoc(uint32_t docId) { @@ -365,7 +349,6 @@ FieldInverter::abortPendingDoc(uint32_t docId) } } - void FieldInverter::moveNotAbortedDocs(uint32_t &dstIdx, uint32_t srcIdx, @@ -390,7 +373,6 @@ FieldInverter::moveNotAbortedDocs(uint32_t &dstIdx, dstIdx += size; } - void FieldInverter::trimAbortedDocs() { @@ -413,7 +395,6 @@ FieldInverter::trimAbortedDocs() _abortedDocs.clear(); } - void FieldInverter::invertField(uint32_t docId, const FieldValue::UP &val) { @@ -424,7 +405,6 @@ FieldInverter::invertField(uint32_t docId, const FieldValue::UP &val) endDoc(); } - void FieldInverter::invertNormalDocTextField(const FieldValue &val) { @@ -467,7 +447,6 @@ FieldInverter::invertNormalDocTextField(const FieldValue &val) } } - namespace { struct FullRadix { @@ -479,9 +458,8 @@ struct FullRadix { } - void -FieldInverter::applyRemoves(DocumentRemover &remover) +FieldInverter::applyRemoves(FieldIndexRemover &remover) { for (auto docId : _removeDocs) { remover.remove(docId, *this); @@ -489,9 +467,8 @@ FieldInverter::applyRemoves(DocumentRemover &remover) _removeDocs.clear(); } - void -FieldInverter::pushDocuments(IOrderedDocumentInserter &inserter) +FieldInverter::pushDocuments(IOrderedFieldIndexInserter &inserter) { trimAbortedDocs(); @@ -568,6 +545,5 @@ FieldInverter::pushDocuments(IOrderedDocumentInserter &inserter) reset(); } - } diff --git a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.h b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h index 69cfd370041..ecf2f8d8979 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/fieldinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/field_inverter.h @@ -2,27 +2,25 @@ #pragma once -#include <map> -#include <set> -#include <vespa/document/fieldvalue/document.h> +#include "i_field_index_remove_listener.h" +#include <vespa/document/annotation/span.h> #include <vespa/document/datatype/datatypes.h> -#include <limits> -#include "i_document_remove_listener.h" -#include <vespa/searchlib/index/docidandfeatures.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> -#include <vespa/document/annotation/span.h> +#include <vespa/searchlib/index/docidandfeatures.h> +#include <limits> +#include <map> +#include <set> namespace search::memoryindex { -class IOrderedDocumentInserter; -class DocumentRemover; +class IOrderedFieldIndexInserter; +class FieldIndexRemover; -class FieldInverter : public IDocumentRemoveListener -{ +class FieldInverter : public IFieldIndexRemoveListener { public: - class PosInfo - { + class PosInfo { public: uint32_t _wordNum; // XXX: Initially word reference uint32_t _docId; @@ -54,7 +52,6 @@ public: { } - PosInfo(uint32_t wordRef, uint32_t docId) : _wordNum(wordRef), @@ -65,22 +62,19 @@ public: { } - bool - removed() const - { - return _elemId == _elemRemoved; - } + bool removed() const { return _elemId == _elemRemoved; } - bool - operator<(const PosInfo &rhs) const - { - if (_wordNum != rhs._wordNum) + bool operator<(const PosInfo &rhs) const { + if (_wordNum != rhs._wordNum) { return _wordNum < rhs._wordNum; - if (_docId != rhs._docId) + } + if (_docId != rhs._docId) { return _docId < rhs._docId; + } if (_elemId != rhs._elemId) { - if (removed() != rhs.removed()) + if (removed() != rhs.removed()) { return removed() && !rhs.removed(); + } return _elemId < rhs._elemId; } return _wordPos < rhs._wordPos; @@ -95,8 +89,7 @@ private: using WordBuffer = vespalib::Array<char>; - class ElemInfo - { + class ElemInfo { public: int32_t _weight; uint32_t _len; @@ -107,18 +100,13 @@ private: { } - void - setLen(uint32_t len) - { - _len = len; - } + void setLen(uint32_t len) { _len = len; } }; using ElemInfoVec = std::vector<ElemInfo>; using PosInfoVec = std::vector<PosInfo>; - class CompareWordRef - { + class CompareWordRef { const char *const _wordBuffer; public: @@ -127,15 +115,11 @@ private: { } - const char * - getWord(uint32_t wordRef) const - { + const char *getWord(uint32_t wordRef) const { return &_wordBuffer[static_cast<size_t>(wordRef) << 2]; } - bool - operator()(const uint32_t lhs, const uint32_t rhs) const - { + bool operator()(const uint32_t lhs, const uint32_t rhs) const { return strcmp(getWord(lhs), getWord(rhs)) < 0; } }; @@ -143,8 +127,7 @@ private: /* * Range in _positions vector used to represent a document put. */ - class PositionRange - { + class PositionRange { uint32_t _start; uint32_t _len; @@ -155,9 +138,7 @@ private: { } - bool - operator<(const PositionRange &rhs) const - { + bool operator<(const PositionRange &rhs) const { if (_start != rhs._start) { return _start < rhs._start; } @@ -202,14 +183,12 @@ public: * * @param weight element weight */ - void - startElement(int32_t weight); + void startElement(int32_t weight); /** * End an element. */ - void - endElement(); + void endElement(); private: /** @@ -220,8 +199,7 @@ private: * * @return word reference */ - VESPA_DLL_LOCAL uint32_t - saveWord(const vespalib::stringref word); + VESPA_DLL_LOCAL uint32_t saveWord(const vespalib::stringref word); /** * Save field value as word in word buffer. @@ -230,8 +208,7 @@ private: * * @return word reference */ - VESPA_DLL_LOCAL uint32_t - saveWord(const document::FieldValue &fv); + VESPA_DLL_LOCAL uint32_t saveWord(const document::FieldValue &fv); /** * Get pointer to saved word from a word reference. @@ -240,9 +217,7 @@ private: * * @return saved word */ - const char * - getWordFromRef(uint32_t wordRef) const - { + const char *getWordFromRef(uint32_t wordRef) const { return &_words[static_cast<size_t>(wordRef) << 2]; } @@ -253,9 +228,7 @@ private: * * @return saved word */ - const char * - getWordFromNum(uint32_t wordNum) const - { + const char *getWordFromNum(uint32_t wordNum) const { return getWordFromRef(_wordRefs[wordNum]); } @@ -266,9 +239,7 @@ private: * * @return word number */ - uint32_t - getWordNum(uint32_t wordRef) const - { + uint32_t getWordNum(uint32_t wordRef) const { const char *p = &_words[static_cast<size_t>(wordRef - 1) << 2]; return *reinterpret_cast<const uint32_t *>(p); } @@ -279,9 +250,7 @@ private: * @param wordRef word reference * @param wordNum word number */ - void - updateWordNum(uint32_t wordRef, uint32_t wordNum) - { + void updateWordNum(uint32_t wordRef, uint32_t wordNum) { char *p = &_words[static_cast<size_t>(wordRef - 1) << 2]; *reinterpret_cast<uint32_t *>(p) = wordNum; } @@ -292,17 +261,12 @@ private: * * @param wordRef word reference */ - void - add(uint32_t wordRef) { + void add(uint32_t wordRef) { _positions.emplace_back(wordRef, _docId, _elem, _wpos, _elems.size() - 1); } - void - stepWordPos() - { - ++_wpos; - } + void stepWordPos() { ++_wpos; } public: VESPA_DLL_LOCAL void @@ -323,30 +287,22 @@ private: * * @return schema used by this index */ - const index::Schema & - getSchema() const - { - return _schema; - } + const index::Schema &getSchema() const { return _schema; } /** * Clear internal memory structures. */ - void - reset(); + void reset(); /** * Calculate word numbers and replace word references with word * numbers in internal memory structures. */ - void - sortWords(); + void sortWords(); - void - moveNotAbortedDocs(uint32_t &dstIdx, uint32_t srcIdx, uint32_t nextTrimIdx); + void moveNotAbortedDocs(uint32_t &dstIdx, uint32_t srcIdx, uint32_t nextTrimIdx); - void - trimAbortedDocs(); + void trimAbortedDocs(); /* * Abort a pending document that has already been inverted. @@ -354,8 +310,7 @@ private: * @param docId local id for document * */ - void - abortPendingDoc(uint32_t docId); + void abortPendingDoc(uint32_t docId); public: /** @@ -371,42 +326,31 @@ public: * * @param remover document remover */ - void - applyRemoves(DocumentRemover &remover); + void applyRemoves(FieldIndexRemover &remover); /** - * Push inverted documents to memory index structure. + * Push inverted documents to field index structure using the given inserter. * - * Temporary restriction: Currently only one document at a time is - * supported. - * - * @param inserter ordered document inserter + * Temporary restriction: Currently only one document at a time is supported. */ - void - pushDocuments(IOrderedDocumentInserter &inserter); + void pushDocuments(IOrderedFieldIndexInserter &inserter); /* * Invert a normal text field, based on annotations. */ - void - invertField(uint32_t docId, const document::FieldValue::UP &val); + void invertField(uint32_t docId, const document::FieldValue::UP &val); /* * Setup remove of word in old version of document. */ - virtual void - remove(const vespalib::stringref word, uint32_t docId) override; + virtual void remove(const vespalib::stringref word, uint32_t docId) override; - void - removeDocument(uint32_t docId) - { + void removeDocument(uint32_t docId) { abortPendingDoc(docId); _removeDocs.push_back(docId); } - void - startDoc(uint32_t docId) - { + void startDoc(uint32_t docId) { assert(_docId == 0); assert(docId != 0); abortPendingDoc(docId); @@ -416,9 +360,7 @@ public: _wpos = 0; } - void - endDoc() - { + void endDoc() { uint32_t newPosSize = static_cast<uint32_t>(_positions.size()); _pendingDocs.insert({ _docId, { _oldPosSize, newPosSize - _oldPosSize } }); @@ -426,9 +368,7 @@ public: _oldPosSize = newPosSize; } - void - addWord(const vespalib::stringref word) - { + void addWord(const vespalib::stringref word) { uint32_t wordRef = saveWord(word); if (wordRef != 0u) { add(wordRef); diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_document_insert_listener.h b/searchlib/src/vespa/searchlib/memoryindex/i_document_insert_listener.h deleted file mode 100644 index 194a98ef8ba..00000000000 --- a/searchlib/src/vespa/searchlib/memoryindex/i_document_insert_listener.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once -#include <vespa/searchlib/datastore/entryref.h> - -namespace search::memoryindex { - -/** - * Interface used to track which {wordRef, fieldId} pairs that are - * inserted into the memory index dictionary for a document. - */ -class IDocumentInsertListener -{ -public: - virtual ~IDocumentInsertListener() {} - virtual void insert(datastore::EntryRef wordRef, uint32_t docId) = 0; - virtual void flush() = 0; -}; - -} - diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_field_index_insert_listener.h b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_insert_listener.h new file mode 100644 index 00000000000..0aacfa53c34 --- /dev/null +++ b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_insert_listener.h @@ -0,0 +1,26 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <vespa/searchlib/datastore/entryref.h> + +namespace search::memoryindex { + +/** + * Interface used to track which {wordRef, docId} pairs that are inserted into a FieldIndex. + */ +class IFieldIndexInsertListener { +public: + virtual ~IFieldIndexInsertListener() {} + + /** + * Called when a {wordRef, docId} tuple is inserted into the field index. + */ + virtual void insert(datastore::EntryRef wordRef, uint32_t docId) = 0; + + /** + * Called to process the set of {wordRef, docId} tuples inserted since last flush(). + */ + virtual void flush() = 0; +}; + +} + diff --git a/searchlib/src/vespa/searchlib/memoryindex/i_document_remove_listener.h b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_remove_listener.h index 436ee0a49e3..4419303a654 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/i_document_remove_listener.h +++ b/searchlib/src/vespa/searchlib/memoryindex/i_field_index_remove_listener.h @@ -7,13 +7,15 @@ namespace search::memoryindex { /** - * Interface used to track which {wordRef, fieldId} pairs that are - * removed from the memory index dictionary for a document. + * Interface used to track which {word, docId} pairs that are removed from a FieldIndex. */ -class IDocumentRemoveListener -{ +class IFieldIndexRemoveListener { public: - virtual ~IDocumentRemoveListener() {} + virtual ~IFieldIndexRemoveListener() {} + + /** + * Called when a {word, docId} tuple is removed from the field index. + */ virtual void remove(const vespalib::stringref word, uint32_t docId) = 0; }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/iordereddocumentinserter.h b/searchlib/src/vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h index 9edd1eb4d3b..a1eee2e10ee 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/iordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h @@ -10,13 +10,14 @@ namespace search::index { class DocIdAndFeatures; } namespace search::memoryindex { /** - * Interface class for ordered document inserter. + * Interface used to insert inverted documents into a FieldIndex, + * updating the underlying posting lists in that index. * - * Insert order must be properly sorted, by (word, docId) + * Insert order must be properly sorted, first by word, then by docId. */ -class IOrderedDocumentInserter { +class IOrderedFieldIndexInserter { public: - virtual ~IOrderedDocumentInserter() {} + virtual ~IOrderedFieldIndexInserter() {} /** * Set next word to operate on. @@ -24,7 +25,7 @@ public: virtual void setNextWord(const vespalib::stringref word) = 0; /** - * Add (word, docId) tuple with given features. + * Add (word, docId) tuple with the given features. */ virtual void add(uint32_t docId, const index::DocIdAndFeatures &features) = 0; @@ -33,15 +34,13 @@ public: */ virtual void remove(uint32_t docId) = 0; - /* - * Flush pending changes to postinglist for (_word). - * - * _dItr is located at correct position. + /** + * Flush pending changes for the current word (into the underlying posting list). */ virtual void flush() = 0; - /* - * Rewind iterator, to start new pass. + /** + * Rewind to prepare for another set of (word, docId) tuples. */ virtual void rewind() = 0; }; diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index 90036c83efb..3ff2d553a96 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -1,9 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "documentinverter.h" +#include "document_inverter.h" #include "field_index_collection.h" -#include "memoryindex.h" -#include "postingiterator.h" +#include "memory_index.h" +#include "posting_iterator.h" #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/searchlib/btree/btreenodeallocator.hpp> @@ -15,7 +15,7 @@ #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/log/log.h> -LOG_SETUP(".searchlib.memoryindex.memoryindex"); +LOG_SETUP(".searchlib.memoryindex.memory_index"); using document::ArrayFieldValue; using document::WeightedSetFieldValue; @@ -118,7 +118,6 @@ MemoryIndex::commit(const std::shared_ptr<IDestructorCallback> &onWriteDone) flipInverter(); } - void MemoryIndex::flipInverter() { @@ -139,8 +138,7 @@ MemoryIndex::dump(IndexBuilder &indexBuilder) namespace { -class MemTermBlueprint : public queryeval::SimpleLeafBlueprint -{ +class MemTermBlueprint : public queryeval::SimpleLeafBlueprint { private: GenerationHandler::Guard _genGuard; FieldIndex::PostingList::ConstIterator _pitr; @@ -167,8 +165,7 @@ public: setEstimate(estimate); } - SearchIterator::UP - createLeafSearch(const TermFieldMatchDataArray &tfmda, bool) const override { + SearchIterator::UP createLeafSearch(const TermFieldMatchDataArray &tfmda, bool) const override { auto search = std::make_unique<PostingIterator>(_pitr, _featureStore, _fieldId, tfmda); if (_useBitVector) { LOG(debug, "Return BooleanMatchIteratorWrapper: fieldId(%u), docCount(%zu)", @@ -185,8 +182,7 @@ public: /** * Determines the correct Blueprint to use. **/ -class CreateBlueprintVisitor : public CreateBlueprintVisitorHelper -{ +class CreateBlueprintVisitor : public CreateBlueprintVisitorHelper { private: const FieldSpec &_field; const uint32_t _fieldId; diff --git a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index 621c72d56a3..0b74e05c619 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memoryindex.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -23,8 +23,7 @@ class FieldIndexCollection; * Lock-free implementation of a memory-based index * using the document inverter and dictionary classes from searchlib. **/ -class MemoryIndex : public queryeval::Searchable -{ +class MemoryIndex : public queryeval::Searchable { private: index::Schema _schema; ISequencedTaskExecutor &_invertThreads; @@ -136,15 +135,13 @@ public: void dump(index::IndexBuilder &indexBuilder); // implements Searchable - queryeval::Blueprint::UP - createBlueprint(const queryeval::IRequestContext & requestContext, - const queryeval::FieldSpec &field, - const query::Node &term) override; - - queryeval::Blueprint::UP - createBlueprint(const queryeval::IRequestContext & requestContext, - const queryeval::FieldSpecList &fields, - const query::Node &term) override { + queryeval::Blueprint::UP createBlueprint(const queryeval::IRequestContext & requestContext, + const queryeval::FieldSpec &field, + const query::Node &term) override; + + queryeval::Blueprint::UP createBlueprint(const queryeval::IRequestContext & requestContext, + const queryeval::FieldSpecList &fields, + const query::Node &term) override { return queryeval::Searchable::createBlueprint(requestContext, fields, term); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.cpp b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp index 3c4fca5b044..9b127a8b096 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.cpp @@ -1,12 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "ordereddocumentinserter.h" -#include "i_document_insert_listener.h" +#include "i_field_index_insert_listener.h" +#include "ordered_field_index_inserter.h" #include <vespa/searchlib/index/docidandfeatures.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreenodeallocator.hpp> @@ -27,7 +27,7 @@ const vespalib::string emptyWord = ""; } -OrderedDocumentInserter::OrderedDocumentInserter(FieldIndex &fieldIndex) +OrderedFieldIndexInserter::OrderedFieldIndexInserter(FieldIndex &fieldIndex) : _word(), _prevDocId(noDocId), _prevAdd(false), @@ -39,14 +39,13 @@ OrderedDocumentInserter::OrderedDocumentInserter(FieldIndex &fieldIndex) { } -OrderedDocumentInserter::~OrderedDocumentInserter() +OrderedFieldIndexInserter::~OrderedFieldIndexInserter() { flush(); } - void -OrderedDocumentInserter::flushWord() +OrderedFieldIndexInserter::flushWord() { if (_removes.empty() && _adds.empty()) { return; @@ -68,17 +67,15 @@ OrderedDocumentInserter::flushWord() _adds.clear(); } - void -OrderedDocumentInserter::flush() +OrderedFieldIndexInserter::flush() { flushWord(); _listener.flush(); } - void -OrderedDocumentInserter::setNextWord(const vespalib::stringref word) +OrderedFieldIndexInserter::setNextWord(const vespalib::stringref word) { // TODO: Adjust here if zero length words should be legal. assert(_word < word); @@ -103,10 +100,9 @@ OrderedDocumentInserter::setNextWord(const vespalib::stringref word) assert(_word == wordStore.getWord(_dItr.getKey()._wordRef)); } - void -OrderedDocumentInserter::add(uint32_t docId, - const index::DocIdAndFeatures &features) +OrderedFieldIndexInserter::add(uint32_t docId, + const index::DocIdAndFeatures &features) { assert(docId != noDocId); assert(_prevDocId == noDocId || _prevDocId < docId || @@ -118,9 +114,8 @@ OrderedDocumentInserter::add(uint32_t docId, _prevAdd = true; } - void -OrderedDocumentInserter::remove(uint32_t docId) +OrderedFieldIndexInserter::remove(uint32_t docId) { assert(docId != noDocId); assert(_prevDocId == noDocId || _prevDocId < docId); @@ -129,9 +124,8 @@ OrderedDocumentInserter::remove(uint32_t docId) _prevAdd = false; } - void -OrderedDocumentInserter::rewind() +OrderedFieldIndexInserter::rewind() { assert(_removes.empty() && _adds.empty()); _word = ""; @@ -140,9 +134,8 @@ OrderedDocumentInserter::rewind() _dItr.begin(); } - datastore::EntryRef -OrderedDocumentInserter::getWordRef() const +OrderedFieldIndexInserter::getWordRef() const { return _dItr.getKey()._wordRef; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.h b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h index 328346e9eee..03cf3723f01 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/ordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/ordered_field_index_inserter.h @@ -2,23 +2,25 @@ #pragma once -#include "iordereddocumentinserter.h" +#include "i_ordered_field_index_inserter.h" #include "field_index.h" #include <limits> namespace search::memoryindex { -class IDocumentInsertListener; - +class IFieldIndexInsertListener; /** - * Class for inserting updates to FieldIndex in an ordered manner - * (single pass scan of dictionary tree) + * Class used to insert inverted documents into a FieldIndex, + * updating the underlying posting lists in that index. + * + * This is done by doing a single pass scan of the dictionary of the FieldIndex, + * and for each word updating the posting list with docId adds / removes. * - * Insert order must be properly sorted, by (word, docId) + * Insert order must be properly sorted, first by word, then by docId. */ -class OrderedDocumentInserter : public IOrderedDocumentInserter -{ +class OrderedFieldIndexInserter : public IOrderedFieldIndexInserter { +private: vespalib::stringref _word; uint32_t _prevDocId; bool _prevAdd; @@ -29,7 +31,7 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter using PostingListKeyDataType = FieldIndex::PostingListKeyDataType; FieldIndex &_fieldIndex; DictionaryTree::Iterator _dItr; - IDocumentInsertListener &_listener; + IFieldIndexInsertListener &_listener; // Pending changes to posting list for (_word) std::vector<uint32_t> _removes; @@ -39,7 +41,7 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter static constexpr uint32_t noFieldId = std::numeric_limits<uint32_t>::max(); static constexpr uint32_t noDocId = std::numeric_limits<uint32_t>::max(); - /* + /** * Flush pending changes to postinglist for (_word). * * _dItr is located at correct position. @@ -47,13 +49,13 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter void flushWord(); public: - OrderedDocumentInserter(FieldIndex &fieldIndex); - ~OrderedDocumentInserter() override; + OrderedFieldIndexInserter(FieldIndex &fieldIndex); + ~OrderedFieldIndexInserter() override; void setNextWord(const vespalib::stringref word) override; void add(uint32_t docId, const index::DocIdAndFeatures &features) override; void remove(uint32_t docId) override; - /* + /** * Flush pending changes to postinglist for (_word). Also flush * insert listener. * @@ -61,7 +63,7 @@ public: */ void flush() override; - /* + /** * Rewind iterator, to start new pass. */ void rewind() override; diff --git a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.cpp b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp index ca56299f906..4c29ec321e3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.cpp @@ -1,14 +1,14 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "postingiterator.h" +#include "posting_iterator.h" +#include <vespa/searchlib/btree/btreeiterator.hpp> #include <vespa/searchlib/btree/btreenode.hpp> #include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenodestore.hpp> -#include <vespa/searchlib/btree/btreeiterator.hpp> #include <vespa/searchlib/btree/btreeroot.hpp> #include <vespa/log/log.h> -LOG_SETUP(".searchlib.memoryindex.postingiterator"); +LOG_SETUP(".searchlib.memoryindex.posting_iterator"); namespace search::memoryindex { diff --git a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.h b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h index 2838c65c5eb..de337ef49f3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/postingiterator.h +++ b/searchlib/src/vespa/searchlib/memoryindex/posting_iterator.h @@ -10,8 +10,7 @@ namespace search::memoryindex { /** * Search iterator for memory field index posting list. */ -class PostingIterator : public queryeval::RankedSearchIteratorBase -{ +class PostingIterator : public queryeval::RankedSearchIteratorBase { private: FieldIndex::PostingList::ConstIterator _itr; const FeatureStore &_featureStore; diff --git a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp index 2c290f17782..c185ec93c9d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.cpp @@ -1,20 +1,20 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "urlfieldinverter.h" -#include "fieldinverter.h" +#include "field_inverter.h" +#include "url_field_inverter.h" #include <vespa/document/datatype/urldatatype.h> #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/stringfieldvalue.h> #include <vespa/document/fieldvalue/weightedsetfieldvalue.h> +#include <vespa/searchlib/common/sort.h> #include <vespa/searchlib/util/url.h> -#include <stdexcept> -#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/text/lowercase.h> +#include <vespa/vespalib/text/utf8.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/searchlib/common/sort.h> +#include <stdexcept> #include <vespa/log/log.h> -LOG_SETUP(".memoryindex.urlfieldinverter"); +LOG_SETUP(".memoryindex.url_field_inverter"); namespace search::memoryindex { @@ -46,7 +46,6 @@ lowercaseToken(vespalib::string &dest, const char *src, size_t srcSize) } - using document::ArrayFieldValue; using document::DataType; using document::FieldValue; @@ -61,7 +60,6 @@ using search::index::schema::CollectionType; using search::util::URL; using vespalib::make_string; - void UrlFieldInverter::startDoc(uint32_t docId) { @@ -75,7 +73,6 @@ UrlFieldInverter::startDoc(uint32_t docId) _hostname->startDoc(docId); } - void UrlFieldInverter::endDoc() { @@ -89,7 +86,6 @@ UrlFieldInverter::endDoc() _hostname->endDoc(); } - void UrlFieldInverter::startElement(int32_t weight) { @@ -103,7 +99,6 @@ UrlFieldInverter::startElement(int32_t weight) _hostname->startElement(weight); } - void UrlFieldInverter::endElement() { @@ -117,7 +112,6 @@ UrlFieldInverter::endElement() _hostname->endElement(); } - void UrlFieldInverter::processUrlSubField(FieldInverter *inverter, const StructFieldValue &field, @@ -145,7 +139,6 @@ UrlFieldInverter::processUrlSubField(FieldInverter *inverter, } } - void UrlFieldInverter::processAnnotatedUrlField(const StructFieldValue & field) { @@ -159,7 +152,6 @@ UrlFieldInverter::processAnnotatedUrlField(const StructFieldValue & field) processUrlSubField(_hostname, field, UrlDataType::FIELD_HOST, true); } - void UrlFieldInverter::processUrlField(const FieldValue &url_field) { @@ -207,7 +199,9 @@ UrlFieldInverter::processUrlField(const FieldValue &url_field) processUrlOldStyle(s); } -void UrlFieldInverter::processUrlOldStyle(const vespalib::string &s) { +void +UrlFieldInverter::processUrlOldStyle(const vespalib::string &s) +{ URL url(reinterpret_cast<const unsigned char *>(s.data()), s.size()); _hostname->addWord(HOSTNAME_BEGIN); @@ -264,7 +258,6 @@ void UrlFieldInverter::processUrlOldStyle(const vespalib::string &s) { _hostname->addWord(HOSTNAME_END); } - void UrlFieldInverter::processArrayUrlField(const ArrayFieldValue &field) { @@ -276,7 +269,6 @@ UrlFieldInverter::processArrayUrlField(const ArrayFieldValue &field) } } - void UrlFieldInverter::processWeightedSetUrlField(const WeightedSetFieldValue &field) { @@ -292,13 +284,16 @@ UrlFieldInverter::processWeightedSetUrlField(const WeightedSetFieldValue &field) } namespace { -bool isUriType(const DataType &type) { + +bool +isUriType(const DataType &type) +{ return type == UrlDataType::getInstance() - || type == *DataType::STRING - || type == *DataType::URI; + || type == *DataType::STRING + || type == *DataType::URI; } -} // namespace +} void UrlFieldInverter::invertUrlField(const FieldValue &val) @@ -366,7 +361,6 @@ UrlFieldInverter::removeDocument(uint32_t docId) _hostname->removeDocument(docId); } - UrlFieldInverter::UrlFieldInverter(index::Schema::CollectionType collectionType, FieldInverter *all, FieldInverter *scheme, @@ -389,6 +383,5 @@ UrlFieldInverter::UrlFieldInverter(index::Schema::CollectionType collectionType, { } - } diff --git a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.h b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.h index c902feaf5a6..1659e460af3 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/urlfieldinverter.h +++ b/searchlib/src/vespa/searchlib/memoryindex/url_field_inverter.h @@ -9,8 +9,7 @@ namespace search::memoryindex { class FieldInverter; -class UrlFieldInverter -{ +class UrlFieldInverter { FieldInverter *_all; FieldInverter *_scheme; FieldInverter *_host; @@ -31,11 +30,10 @@ class UrlFieldInverter void endElement(); - void - processUrlSubField(FieldInverter *inverter, - const document::StructFieldValue &field, - vespalib::stringref subField, - bool addAnchors); + void processUrlSubField(FieldInverter *inverter, + const document::StructFieldValue &field, + vespalib::stringref subField, + bool addAnchors); void processAnnotatedUrlField(const document::StructFieldValue &field); diff --git a/searchlib/src/vespa/searchlib/memoryindex/wordstore.cpp b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp index b65fe192e58..ffdc26f5eb0 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/wordstore.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp @@ -1,6 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "wordstore.h" +#include "word_store.h" #include <vespa/searchlib/datastore/datastore.hpp> namespace search::memoryindex { diff --git a/searchlib/src/vespa/searchlib/memoryindex/wordstore.h b/searchlib/src/vespa/searchlib/memoryindex/word_store.h index b909f26157f..4c1526df527 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/wordstore.h +++ b/searchlib/src/vespa/searchlib/memoryindex/word_store.h @@ -7,8 +7,7 @@ namespace search::memoryindex { -class WordStore -{ +class WordStore { public: using DataStoreType = datastore::DataStoreT<datastore::AlignedEntryRefT<22, 2>>; using RefType = DataStoreType::RefType; @@ -23,8 +22,7 @@ public: WordStore(); ~WordStore(); datastore::EntryRef addWord(const vespalib::stringref word); - const char * getWord(datastore::EntryRef ref) const - { + const char *getWord(datastore::EntryRef ref) const { RefType internalRef(ref); return _store.getEntry<char>(internalRef); } diff --git a/searchlib/src/vespa/searchlib/predicate/document_features_store.h b/searchlib/src/vespa/searchlib/predicate/document_features_store.h index 2cf3e15337a..4c55b67cb11 100644 --- a/searchlib/src/vespa/searchlib/predicate/document_features_store.h +++ b/searchlib/src/vespa/searchlib/predicate/document_features_store.h @@ -4,7 +4,7 @@ #include "predicate_tree_annotator.h" #include <vespa/searchlib/btree/btree.h> -#include <vespa/searchlib/memoryindex/wordstore.h> +#include <vespa/searchlib/memoryindex/word_store.h> #include <vespa/vespalib/data/databuffer.h> #include <vespa/vespalib/stllike/hash_map.h> #include <unordered_set> diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp index 2d7a9abbf79..9cbbd136148 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.cpp @@ -2,13 +2,13 @@ #include "fakememtreeocc.h" #include "fpfactory.h" -#include <vespa/searchlib/queryeval/iterators.h> -#include <vespa/searchlib/btree/btreeroot.hpp> #include <vespa/searchlib/btree/btreeiterator.hpp> -#include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenode.hpp> +#include <vespa/searchlib/btree/btreenodeallocator.hpp> #include <vespa/searchlib/btree/btreenodestore.hpp> -#include <vespa/searchlib/memoryindex/postingiterator.h> +#include <vespa/searchlib/btree/btreeroot.hpp> +#include <vespa/searchlib/memoryindex/posting_iterator.h> +#include <vespa/searchlib/queryeval/iterators.h> #include <vespa/searchlib/util/postingpriorityqueue.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h index 7fa46fc7531..f0363500559 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakememtreeocc.h @@ -4,7 +4,7 @@ #include "fakeword.h" #include "fakeposting.h" #include "fpfactory.h" -#include <vespa/searchlib/memoryindex/featurestore.h> +#include <vespa/searchlib/memoryindex/feature_store.h> #include <vespa/searchlib/memoryindex/field_index.h> #include <vespa/searchlib/bitcompression/compression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp index 3c16fc8e9a8..33819d4f7cb 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.cpp @@ -3,12 +3,13 @@ #include "fakezcfilterocc.h" #include "fpfactory.h" #include <vespa/searchlib/diskindex/zcposocciterators.h> -#include <vespa/searchlib/diskindex/zcbuf.h> +#include <vespa/searchlib/diskindex/zc4_posting_writer.h> using search::fef::TermFieldMatchData; using search::fef::TermFieldMatchDataArray; using search::fef::TermFieldMatchDataPosition; using search::queryeval::SearchIterator; +using search::index::PostingListCounts; using search::index::PostingListParams; using search::index::DocIdAndFeatures; using search::index::DocIdAndPosOccFeatures; @@ -24,11 +25,6 @@ namespace search { namespace fakedata { -#define L1SKIPSTRIDE 16 -#define L2SKIPSTRIDE 8 -#define L3SKIPSTRIDE 8 -#define L4SKIPSTRIDE 8 - #define DEBUG_ZCFILTEROCC_PRINTF 0 #define DEBUG_ZCFILTEROCC_ASSERT 0 @@ -137,35 +133,8 @@ void FakeZcFilterOcc::setupT(const FakeWord &fw, bool doFeatures, bool dynamicK) { - ZcBuf bytes; - ZcBuf l1SkipBytes; - ZcBuf l2SkipBytes; - ZcBuf l3SkipBytes; - ZcBuf l4SkipBytes; - uint32_t lastDocId = 0u; - uint32_t lastL1SkipDocId = 0u; - uint64_t lastL1SkipDocIdPos = 0; - uint64_t lastL1SkipFeaturePos = 0; - unsigned int l1SkipCnt = 0; - uint32_t lastL2SkipDocId = 0u; - uint64_t lastL2SkipDocIdPos = 0; - uint64_t lastL2SkipFeaturePos = 0; - uint64_t lastL2SkipL1SkipPos = 0; - unsigned int l2SkipCnt = 0; - uint32_t lastL3SkipDocId = 0u; - uint64_t lastL3SkipDocIdPos = 0; - uint64_t lastL3SkipFeaturePos = 0; - uint64_t lastL3SkipL1SkipPos = 0; - uint64_t lastL3SkipL2SkipPos = 0; - unsigned int l3SkipCnt = 0; - uint32_t lastL4SkipDocId = 0u; - uint64_t lastL4SkipDocIdPos = 0; - uint64_t lastL4SkipFeaturePos = 0; - uint64_t lastL4SkipL1SkipPos = 0; - uint64_t lastL4SkipL2SkipPos = 0; - uint64_t lastL4SkipL3SkipPos = 0; - unsigned int l4SkipCnt = 0; - uint64_t featurePos = 0; + PostingListCounts counts; + Zc4PostingWriter<bigEndian> writer(counts); typedef FakeWord FW; typedef FW::DocWordFeatureList DWFL; @@ -181,288 +150,88 @@ FakeZcFilterOcc::setupT(const FakeWord &fw, bool doFeatures, FeatureEncodeContext<bigEndian> &f = (dynamicK ? static_cast<FeatureEncodeContext<bigEndian> &>(f1) : static_cast<FeatureEncodeContext<bigEndian> &>(f0)); - search::ComprFileWriteContext fctx(f); - f.setWriteContext(&fctx); - fctx.allocComprBuf(64, 1); - f.afterWrite(fctx, 0, 0); + writer.set_dynamic_k(dynamicK); + if (doFeatures) { + writer.set_encode_features(&f); + } + PostingListParams params; + params.set("docIdLimit", fw._docIdLimit); + params.set("minChunkDocs", 1000000000); // Disable chunking + params.set("minSkipDocs", 1u); // Force skip info + writer.set_posting_list_params(params); + auto &writeContext = writer.get_write_context(); + search::ComprBuffer &cb = writeContext; + auto &e = writer.get_encode_context(); + writeContext.allocComprBuf(65536u, 32768u); + e.setupWrite(cb); // Ensure that some space is initially available in encoding buffers - bytes.maybeExpand(); - l1SkipBytes.maybeExpand(); - l2SkipBytes.maybeExpand(); - l3SkipBytes.maybeExpand(); - l4SkipBytes.maybeExpand(); while (d != de) { - if (l1SkipCnt >= L1SKIPSTRIDE) { - uint32_t docIdDelta = lastDocId - lastL1SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - l1SkipBytes.encode(docIdDelta - 1); - uint64_t lastDocIdPos = bytes.size(); - uint32_t docIdPosDelta = lastDocIdPos - lastL1SkipDocIdPos; - l1SkipBytes.encode(docIdPosDelta - 1); - if (doFeatures) { - featurePos = f.getWriteOffset(); - l1SkipBytes.encode(featurePos - lastL1SkipFeaturePos - 1); - lastL1SkipFeaturePos = featurePos; - } -#if DEBUG_ZCFILTEROCC_PRINTF - printf("L1Encode docId=%d (+%d), docIdPos=%d (+%u)\n", - lastDocId, docIdDelta, - (int) lastDocIdPos, docIdPosDelta); -#endif - lastL1SkipDocId = lastDocId; - lastL1SkipDocIdPos = lastDocIdPos; - l1SkipCnt = 0; - ++l2SkipCnt; - if (l2SkipCnt >= L2SKIPSTRIDE) { - docIdDelta = lastDocId - lastL2SkipDocId; - docIdPosDelta = lastDocIdPos - lastL2SkipDocIdPos; - uint64_t lastL1SkipPos = l1SkipBytes.size(); - uint32_t l1SkipPosDelta = lastL1SkipPos - lastL2SkipL1SkipPos; - l2SkipBytes.encode(docIdDelta - 1); - l2SkipBytes.encode(docIdPosDelta - 1); - if (doFeatures) { - l2SkipBytes.encode(featurePos - lastL2SkipFeaturePos - 1); - lastL2SkipFeaturePos = featurePos; - } - l2SkipBytes.encode(l1SkipPosDelta - 1); -#if DEBUG_ZCFILTEROCC_PRINTF - printf("L2Encode docId=%d (+%d), docIdPos=%d (+%u)," - " l1SkipPos=%d (+%u)\n", - lastDocId, docIdDelta, - (int) lastDocIdPos, docIdPosDelta, - (int) lastL1SkipPos, l1SkipPosDelta); -#endif - lastL2SkipDocId = lastDocId; - lastL2SkipDocIdPos = lastDocIdPos; - lastL2SkipL1SkipPos = lastL1SkipPos; - l2SkipCnt = 0; - ++l3SkipCnt; - if (l3SkipCnt >= L3SKIPSTRIDE) { - docIdDelta = lastDocId - lastL3SkipDocId; - docIdPosDelta = lastDocIdPos - lastL3SkipDocIdPos; - l1SkipPosDelta = lastL1SkipPos - lastL3SkipL1SkipPos; - uint64_t lastL2SkipPos = l2SkipBytes.size(); - uint32_t l2SkipPosDelta = lastL2SkipPos - - lastL3SkipL2SkipPos; - l3SkipBytes.encode(docIdDelta - 1); - l3SkipBytes.encode(docIdPosDelta - 1); - if (doFeatures) { - l3SkipBytes.encode(featurePos - lastL3SkipFeaturePos - 1); - lastL3SkipFeaturePos = featurePos; - } - l3SkipBytes.encode(l1SkipPosDelta - 1); - l3SkipBytes.encode(l2SkipPosDelta - 1); -#if DEBUG_ZCFILTEROCC_PRINTF - printf("L3Encode docId=%d (+%d), docIdPos=%d (+%u)," - " l1SkipPos=%d (+%u) l2SkipPos %d (+%u)\n", - lastDocId, docIdDelta, - (int) lastDocIdPos, docIdPosDelta, - (int) lastL1SkipPos, l1SkipPosDelta, - (int) lastL2SkipPos, l2SkipPosDelta); -#endif - lastL3SkipDocId = lastDocId; - lastL3SkipDocIdPos = lastDocIdPos; - lastL3SkipL1SkipPos = lastL1SkipPos; - lastL3SkipL2SkipPos = lastL2SkipPos; - l3SkipCnt = 0; - ++l4SkipCnt; - if (l4SkipCnt >= L4SKIPSTRIDE) { - docIdDelta = lastDocId - lastL4SkipDocId; - docIdPosDelta = lastDocIdPos - lastL4SkipDocIdPos; - l1SkipPosDelta = lastL1SkipPos - lastL4SkipL1SkipPos; - l2SkipPosDelta = lastL2SkipPos - lastL4SkipL2SkipPos; - uint64_t lastL3SkipPos = l3SkipBytes.size(); - uint32_t l3SkipPosDelta = lastL3SkipPos - - lastL4SkipL3SkipPos; - l4SkipBytes.encode(docIdDelta - 1); - l4SkipBytes.encode(docIdPosDelta - 1); - if (doFeatures) { - l4SkipBytes.encode(featurePos - lastL4SkipFeaturePos - 1); - lastL4SkipFeaturePos = featurePos; - } - l4SkipBytes.encode(l1SkipPosDelta - 1); - l4SkipBytes.encode(l2SkipPosDelta - 1); - l4SkipBytes.encode(l3SkipPosDelta - 1); -#if DEBUG_ZCFILTEROCC_PRINTF - printf("L4Encode docId=%d (+%d), docIdPos=%d (+%u)," - " l1SkipPos=%d (+%u) l2SkipPos %d (+%u)" - " l3SkipPos=%d (+%u)\n", - lastDocId, docIdDelta, - (int) lastDocIdPos, docIdPosDelta, - (int) lastL1SkipPos, l1SkipPosDelta, - (int) lastL2SkipPos, l2SkipPosDelta, - (int) lastL3SkipPos, l3SkipPosDelta); -#endif - lastL4SkipDocId = lastDocId; - lastL4SkipDocIdPos = lastDocIdPos; - lastL4SkipL1SkipPos = lastL1SkipPos; - lastL4SkipL2SkipPos = lastL2SkipPos; - lastL4SkipL3SkipPos = lastL3SkipPos; - l4SkipCnt = 0; - } - } - } - } - if (lastDocId == 0u) { - bytes.encode(d->_docId - 1); -#if DEBUG_ZCFILTEROCC_PRINTF - printf("Encode docId=%d\n", - d->_docId); -#endif - } else { - uint32_t docIdDelta = d->_docId - lastDocId; - bytes.encode(docIdDelta - 1); -#if DEBUG_ZCFILTEROCC_PRINTF - printf("Encode docId=%d (+%d)\n", - d->_docId, docIdDelta); -#endif - } if (doFeatures) { fw.setupFeatures(*d, &*p, features); p += d->_positions; - f.writeFeatures(features); + } else { + features.clear(d->_docId); } - lastDocId = d->_docId; - ++l1SkipCnt; + writer.write_docid_and_features(features); ++d; } if (doFeatures) { assert(p == pe); - _featuresSize = f.getWriteOffset(); - // First pad to 64 bits. - uint32_t pad = (64 - f.getWriteOffset()) & 63; - while (pad > 0) { - uint32_t now = std::min(32u, pad); - f.writeBits(0, now); - f.writeComprBufferIfNeeded(); - pad -= now; - } - - // Then write 128 more bits. This allows for 64-bit decoding - // with a readbits that always leaves a nonzero preRead - for (unsigned int i = 0; i < 4; i++) { - f.writeBits(0, 32); - f.writeComprBufferIfNeeded(); - } - f.writeComprBufferIfNeeded(); - f.flush(); - f.writeComprBuffer(); - } else { - _featuresSize = 0; - } - // Extra partial entries for skip tables to simplify iterator during search - if (l1SkipBytes.size() > 0) { - uint32_t docIdDelta = lastDocId - lastL1SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - l1SkipBytes.encode(docIdDelta - 1); - } - if (l2SkipBytes.size() > 0) { - uint32_t docIdDelta = lastDocId - lastL2SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - l2SkipBytes.encode(docIdDelta - 1); - } - if (l3SkipBytes.size() > 0) { - uint32_t docIdDelta = lastDocId - lastL3SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - l3SkipBytes.encode(docIdDelta - 1); - } - if (l4SkipBytes.size() > 0) { - uint32_t docIdDelta = lastDocId - lastL4SkipDocId; - assert(static_cast<int32_t>(docIdDelta) > 0); - l4SkipBytes.encode(docIdDelta - 1); } + writer.flush_word(); + _featuresSize = 0; _hitDocs = fw._postings.size(); _docIdLimit = fw._docIdLimit; - _lastDocId = lastDocId; - FeatureEncodeContext<bigEndian> e; - ComprFileWriteContext ectx(e); - e.setWriteContext(&ectx); - ectx.allocComprBuf(64, 1); - e.afterWrite(ectx, 0, 0); + _compressedBits = e.getWriteOffset(); + assert(_compressedBits == counts._bitLength); + assert(_hitDocs == counts._numDocs); + _lastDocId = fw._postings.back()._docId; + writer.on_close(); - // Encode word header - e.encodeExpGolomb(_hitDocs - 1, K_VALUE_ZCPOSTING_NUMDOCS); - _docIdsSize = bytes.size() * 8; - _l1SkipSize = l1SkipBytes.size(); - _l2SkipSize = _l3SkipSize = _l4SkipSize = 0; - if (_l1SkipSize != 0) - _l2SkipSize = l2SkipBytes.size(); - if (_l2SkipSize != 0) - _l3SkipSize = l3SkipBytes.size(); - if (_l3SkipSize != 0) - _l4SkipSize = l4SkipBytes.size(); - - e.encodeExpGolomb(bytes.size() - 1, K_VALUE_ZCPOSTING_DOCIDSSIZE); - e.encodeExpGolomb(_l1SkipSize, K_VALUE_ZCPOSTING_L1SKIPSIZE); - e.writeComprBufferIfNeeded(); + std::pair<void *, size_t> ectxData = writeContext.grabComprBuffer(_compressedMalloc); + _compressed = std::make_pair(static_cast<uint64_t *>(ectxData.first), + ectxData.second); + read_header<bigEndian>(doFeatures, dynamicK, writer.get_min_skip_docs(), writer.get_min_chunk_docs()); +} + +template <bool bigEndian> +void +FakeZcFilterOcc::read_header(bool doFeatures, bool dynamicK, uint32_t min_skip_docs, uint32_t min_chunk_docs) +{ + // read back word header to get skip sizes + using EC = FeatureEncodeContext<bigEndian>; + UC64_DECODECONTEXT(o); + uint32_t length; + uint64_t val64; + UC64_SETUPBITS_NS(o, _compressed.first, 0, EC); + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_NUMDOCS, EC); + assert(static_cast<uint32_t>(val64) + 1 == _hitDocs); + assert(_hitDocs >= min_skip_docs); + assert(_hitDocs < min_chunk_docs); + uint32_t docIdK = dynamicK ? EC::calcDocIdK(_hitDocs, _docIdLimit) : K_VALUE_ZCPOSTING_LASTDOCID; + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_DOCIDSSIZE, EC); + _docIdsSize = val64 + 1; + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_L1SKIPSIZE, EC); + _l1SkipSize = val64; if (_l1SkipSize != 0) { - e.encodeExpGolomb(_l2SkipSize, K_VALUE_ZCPOSTING_L2SKIPSIZE); - if (_l2SkipSize != 0) { - e.writeComprBufferIfNeeded(); - e.encodeExpGolomb(_l3SkipSize, K_VALUE_ZCPOSTING_L3SKIPSIZE); - if (_l3SkipSize != 0) { - e.encodeExpGolomb(_l4SkipSize, K_VALUE_ZCPOSTING_L4SKIPSIZE); - } - } + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_L2SKIPSIZE, EC); + _l2SkipSize = val64; } - e.writeComprBufferIfNeeded(); - if (doFeatures) { - e.encodeExpGolomb(_featuresSize, K_VALUE_ZCPOSTING_FEATURESSIZE); - } - uint32_t docIdK = e.calcDocIdK(_hitDocs, _docIdLimit); - if (dynamicK) - e.encodeExpGolomb(_docIdLimit - 1 - _lastDocId, docIdK); - else - e.encodeExpGolomb(_docIdLimit - 1 - _lastDocId, - K_VALUE_ZCPOSTING_LASTDOCID); - uint64_t bytePad = (- e.getWriteOffset()) & 7; - if (bytePad > 0) - e.writeBits(0, bytePad); - size_t docIdSize = bytes.size(); - if (docIdSize > 0) { - writeZcBuf(e, bytes); + if (_l2SkipSize != 0) { + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_L3SKIPSIZE, EC); + _l3SkipSize = val64; } - if (_l1SkipSize > 0) { - writeZcBuf(e, l1SkipBytes); - if (_l2SkipSize > 0) { - writeZcBuf(e, l2SkipBytes); - if (_l3SkipSize > 0) { - writeZcBuf(e, l3SkipBytes); - if (_l4SkipSize > 0) { - writeZcBuf(e, l4SkipBytes); - } - } - } + if (_l3SkipSize != 0) { + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_L4SKIPSIZE, EC); + _l4SkipSize = val64; } if (doFeatures) { - e.writeBits(static_cast<const uint64_t *>(fctx._comprBuf), - 0, - _featuresSize); + UC64_DECODEEXPGOLOMB_NS(o, K_VALUE_ZCPOSTING_FEATURESSIZE, EC); + _featuresSize = val64; } - _compressedBits = e.getWriteOffset(); - // First pad to 64 bits. - uint32_t pad = (64 - e.getWriteOffset()) & 63; - while (pad > 0) { - uint32_t now = std::min(32u, pad); - e.writeBits(0, now); - e.writeComprBufferIfNeeded(); - pad -= now; - } - - // Then write 128 more bits. This allows for 64-bit decoding - // with a readbits that always leaves a nonzero preRead - for (unsigned int i = 0; i < 4; i++) { - e.writeBits(0, 32); - e.writeComprBufferIfNeeded(); - } - e.writeComprBufferIfNeeded(); - e.flush(); - e.writeComprBuffer(); - - std::pair<void *, size_t> ectxData = ectx.grabComprBuffer(_compressedMalloc); - _compressed = std::make_pair(static_cast<uint64_t *>(ectxData.first), - ectxData.second); + UC64_DECODEEXPGOLOMB_NS(o, docIdK, EC); + assert(_lastDocId == _docIdLimit - 1 - val64); } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h index d5df198acdc..b68e3866461 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakezcfilterocc.h @@ -37,6 +37,9 @@ protected: template <bool bigEndian> void setupT(const FakeWord &fw, bool doFeatures, bool dynamicK); + template <bool bigEndian> + void read_header(bool do_features, bool dynamic_k, uint32_t min_skip_docs, uint32_t min_cunk_docs); + public: FakeZcFilterOcc(const FakeWord &fw); FakeZcFilterOcc(const FakeWord &fw, bool bigEndian, const char *nameSuffix); diff --git a/searchlib/src/vespa/searchlib/test/memoryindex/ordereddocumentinserter.h b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h index 4802a7571c2..08473f9fc6c 100644 --- a/searchlib/src/vespa/searchlib/test/memoryindex/ordereddocumentinserter.h +++ b/searchlib/src/vespa/searchlib/test/memoryindex/ordered_field_index_inserter.h @@ -2,20 +2,12 @@ #pragma once -#include <vespa/searchlib/memoryindex/iordereddocumentinserter.h> +#include <vespa/searchlib/memoryindex/i_ordered_field_index_inserter.h> #include <sstream> -namespace search -{ +namespace search::memoryindex::test { -namespace memoryindex -{ - -namespace test -{ - -class OrderedDocumentInserter : public IOrderedDocumentInserter -{ +class OrderedFieldIndexInserter : public IOrderedFieldIndexInserter { std::stringstream _ss; bool _first; bool _verbose; @@ -31,7 +23,7 @@ class OrderedDocumentInserter : public IOrderedDocumentInserter } } public: - OrderedDocumentInserter() + OrderedFieldIndexInserter() : _ss(), _first(true), _verbose(false), @@ -115,6 +107,4 @@ public: void setVerbose() { _verbose = true; } }; -} // namespace test -} // namespace memoryindex -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/util/comprfile.cpp b/searchlib/src/vespa/searchlib/util/comprfile.cpp index 26c66f43993..155bb194f97 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.cpp +++ b/searchlib/src/vespa/searchlib/util/comprfile.cpp @@ -409,88 +409,6 @@ ComprFileReadContext::referenceWriteContext(const ComprFileWriteContext &rhs) } -void -ComprFileReadContext::copyWriteContext(const ComprFileWriteContext &rhs) -{ - ComprFileEncodeContext *e = rhs.getEncodeContext(); - ComprFileDecodeContext *d = getDecodeContext(); - - assert(e != NULL); - int usedUnits = e->getUsedUnits(rhs._comprBuf); - assert(usedUnits >= 0); - - dropComprBuf(); - allocComprBuf(usedUnits, 32768); - assert(_comprBufSize >= static_cast<unsigned int>(usedUnits)); - memcpy(_comprBuf, rhs._comprBuf, - static_cast<size_t>(usedUnits) * _unitSize); - setBufferEndFilePos(static_cast<uint64_t>(usedUnits) * _unitSize); - setFileSize(static_cast<uint64_t>(usedUnits) * _unitSize); - if (d != NULL) { - d->afterRead(_comprBuf, - usedUnits, - static_cast<uint64_t>(usedUnits) * _unitSize, - false); - d->setupBits(0); - setBitOffset(-1); - assert(d->getBitPosV() == 0); - } -} - - -void -ComprFileReadContext::referenceReadContext(const ComprFileReadContext &rhs) -{ - ComprFileDecodeContext *d = getDecodeContext(); - - int usedUnits = rhs.getBufferEndFilePos() / _unitSize; - assert(usedUnits >= 0); - assert(static_cast<uint64_t>(usedUnits) * _unitSize == - rhs.getBufferEndFilePos()); - - referenceComprBuf(rhs); - setBufferEndFilePos(static_cast<uint64_t>(usedUnits) * _unitSize); - setFileSize(static_cast<uint64_t>(usedUnits) * _unitSize); - if (d != NULL) { - d->afterRead(_comprBuf, - usedUnits, - static_cast<uint64_t>(usedUnits) * _unitSize, - false); - d->setupBits(0); - setBitOffset(-1); - assert(d->getBitPosV() == 0); - } -} - - -void -ComprFileReadContext::copyReadContext(const ComprFileReadContext &rhs) -{ - ComprFileDecodeContext *d = getDecodeContext(); - - int usedUnits = rhs.getBufferEndFilePos() / _unitSize; - assert(usedUnits >= 0); - assert(static_cast<uint64_t>(usedUnits) * _unitSize == - rhs.getBufferEndFilePos()); - - dropComprBuf(); - allocComprBuf(usedUnits, 32768); - assert(_comprBufSize >= static_cast<unsigned int>(usedUnits)); - memcpy(_comprBuf, rhs._comprBuf, - static_cast<size_t>(usedUnits) * _unitSize); - setBufferEndFilePos(static_cast<uint64_t>(usedUnits) * _unitSize); - setFileSize(static_cast<uint64_t>(usedUnits) * _unitSize); - if (d != NULL) { - d->afterRead(_comprBuf, - usedUnits, - static_cast<uint64_t>(usedUnits) * _unitSize, - false); - d->setupBits(0); - setBitOffset(-1); - assert(d->getBitPosV() == 0); - } -} - ComprFileWriteContext:: ComprFileWriteContext(ComprFileEncodeContext &encodeContext) : ComprBuffer(encodeContext.getUnitByteSize()), diff --git a/searchlib/src/vespa/searchlib/util/comprfile.h b/searchlib/src/vespa/searchlib/util/comprfile.h index 3d44f088c74..d4de1d305fa 100644 --- a/searchlib/src/vespa/searchlib/util/comprfile.h +++ b/searchlib/src/vespa/searchlib/util/comprfile.h @@ -136,23 +136,7 @@ public: * For unit testing only. Reference data owned by rhs, only works as * long as rhs is live and unchanged. */ - void referenceReadContext(const ComprFileReadContext &rhs); - - /* - * For unit testing only. Copy data owned by rhs. - */ - void copyReadContext(const ComprFileReadContext &rhs); - - /* - * For unit testing only. Reference data owned by rhs, only works as - * long as rhs is live and unchanged. - */ void referenceWriteContext(const ComprFileWriteContext &rhs); - - /* - * For unit testing only. Copy data owned by rhs. - */ - void copyWriteContext(const ComprFileWriteContext &rhs); }; diff --git a/storageapi/src/tests/CMakeLists.txt b/storageapi/src/tests/CMakeLists.txt index ebbf3b8357a..ddc43c70004 100644 --- a/storageapi/src/tests/CMakeLists.txt +++ b/storageapi/src/tests/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_executable(storageapi_gtest_runner_app TEST gtest_runner.cpp DEPENDS storageapi_testbuckets + storageapi_testmbusprot storageapi gtest ) @@ -22,8 +23,8 @@ vespa_add_executable(storageapi_testrunner_app TEST testrunner.cpp DEPENDS storageapi_testmessageapi - storageapi_testmbusprot storageapi + vdstestlib ) vespa_add_test( diff --git a/storageapi/src/tests/mbusprot/CMakeLists.txt b/storageapi/src/tests/mbusprot/CMakeLists.txt index 16ced76155c..2801c9a91dd 100644 --- a/storageapi/src/tests/mbusprot/CMakeLists.txt +++ b/storageapi/src/tests/mbusprot/CMakeLists.txt @@ -4,5 +4,5 @@ vespa_add_library(storageapi_testmbusprot storageprotocoltest.cpp DEPENDS storageapi - vdstestlib + gtest ) diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp index f634667afd5..8690d89e12b 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storageapi/src/tests/mbusprot/storageprotocoltest.cpp @@ -14,12 +14,16 @@ #include <vespa/document/update/fieldpathupdates.h> #include <vespa/document/test/make_document_bucket.h> #include <vespa/document/test/make_bucket_space.h> -#include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/vespalib/util/growablebytebuffer.h> #include <vespa/vespalib/objects/nbostream.h> + #include <iomanip> #include <sstream> +#include <gtest/gtest.h> + +using namespace ::testing; + using std::shared_ptr; using document::BucketSpace; using document::ByteBuffer; @@ -32,183 +36,103 @@ using document::test::makeBucketSpace; using storage::lib::ClusterState; using vespalib::string; -namespace storage { -namespace api { +namespace vespalib { + +// Needed for GTest to properly understand how to print Version values. +// If not present, it will print the byte values of the presumed memory area +// (which will be overrun for some reason, causing Valgrind to scream at us). +void PrintTo(const vespalib::Version& v, std::ostream* os) { + *os << v.toString(); +} + +} + +namespace storage::api { -struct StorageProtocolTest : public CppUnit::TestFixture { +struct StorageProtocolTest : TestWithParam<vespalib::Version> { document::TestDocMan _docMan; document::Document::SP _testDoc; document::DocumentId _testDocId; + document::BucketId _bucket_id; document::Bucket _bucket; - vespalib::Version _version5_0{5, 0, 12}; - vespalib::Version _version5_1{5, 1, 0}; - vespalib::Version _version5_2{5, 93, 30}; - vespalib::Version _version6_0{6, 240, 0}; + document::BucketId _dummy_remap_bucket{17, 12345}; + BucketInfo _dummy_bucket_info{1,2,3,4,5, true, false, 48}; documentapi::LoadTypeSet _loadTypes; mbusprot::StorageProtocol _protocol; - static std::vector<std::string> _nonVerboseMessageStrings; - static std::vector<std::string> _verboseMessageStrings; - static std::vector<char> _serialization50; static auto constexpr CONDITION_STRING = "There's just one condition"; StorageProtocolTest() : _docMan(), _testDoc(_docMan.createDocument()), _testDocId(_testDoc->getId()), - _bucket(makeDocumentBucket(document::BucketId(16, 0x51))), + _bucket_id(16, 0x51), + _bucket(makeDocumentBucket(_bucket_id)), _protocol(_docMan.getTypeRepoSP(), _loadTypes) { _loadTypes.addLoadType(34, "foo", documentapi::Priority::PRI_NORMAL_2); } + ~StorageProtocolTest(); + + void set_dummy_bucket_info_reply_fields(BucketInfoReply& reply) { + reply.setBucketInfo(_dummy_bucket_info); + reply.remapBucketId(_dummy_remap_bucket); + } + + void assert_bucket_info_reply_fields_propagated(const BucketInfoReply& reply) { + EXPECT_EQ(_dummy_bucket_info, reply.getBucketInfo()); + EXPECT_TRUE(reply.hasBeenRemapped()); + EXPECT_EQ(_dummy_remap_bucket, reply.getBucketId()); + EXPECT_EQ(_bucket_id, reply.getOriginalBucketId()); + } template<typename Command> - std::shared_ptr<Command> copyCommand(const std::shared_ptr<Command>&, vespalib::Version); + std::shared_ptr<Command> copyCommand(const std::shared_ptr<Command>&); template<typename Reply> std::shared_ptr<Reply> copyReply(const std::shared_ptr<Reply>&); - void recordOutput(const api::StorageMessage& msg); - - void recordSerialization50(); - - void testWriteSerialization50(); - void testAddress50(); - void testStringOutputs(); - - void testPut51(); - void testUpdate51(); - void testGet51(); - void testRemove51(); - void testRevert51(); - void testRequestBucketInfo51(); - void testNotifyBucketChange51(); - void testCreateBucket51(); - void testDeleteBucket51(); - void testMergeBucket51(); - void testGetBucketDiff51(); - void testApplyBucketDiff51(); - void testSplitBucket51(); - void testSplitBucketChain51(); - void testJoinBuckets51(); - void testCreateVisitor51(); - void testDestroyVisitor51(); - void testRemoveLocation51(); - void testInternalMessage(); - void testSetBucketState51(); - - void testPutCommand52(); - void testUpdateCommand52(); - void testRemoveCommand52(); - - void testPutCommandWithBucketSpace6_0(); - void testCreateVisitorWithBucketSpace6_0(); - void testRequestBucketInfoWithBucketSpace6_0(); - - void serialized_size_is_used_to_set_approx_size_of_storage_message(); - - CPPUNIT_TEST_SUITE(StorageProtocolTest); - - // Enable to see string outputs of messages - // CPPUNIT_TEST_DISABLED(testStringOutputs); - - // Enable this to write 5.0 serialization to disk - // CPPUNIT_TEST_DISABLED(testWriteSerialization50); - // CPPUNIT_TEST_DISABLED(testAddress50); - - // 5.1 tests - CPPUNIT_TEST(testPut51); - CPPUNIT_TEST(testUpdate51); - CPPUNIT_TEST(testGet51); - CPPUNIT_TEST(testRemove51); - CPPUNIT_TEST(testRevert51); - CPPUNIT_TEST(testRequestBucketInfo51); - CPPUNIT_TEST(testNotifyBucketChange51); - CPPUNIT_TEST(testCreateBucket51); - CPPUNIT_TEST(testDeleteBucket51); - CPPUNIT_TEST(testMergeBucket51); - CPPUNIT_TEST(testGetBucketDiff51); - CPPUNIT_TEST(testApplyBucketDiff51); - CPPUNIT_TEST(testSplitBucket51); - CPPUNIT_TEST(testJoinBuckets51); - CPPUNIT_TEST(testCreateVisitor51); - CPPUNIT_TEST(testDestroyVisitor51); - CPPUNIT_TEST(testRemoveLocation51); - CPPUNIT_TEST(testInternalMessage); - CPPUNIT_TEST(testSetBucketState51); - - // 5.2 tests - CPPUNIT_TEST(testPutCommand52); - CPPUNIT_TEST(testUpdateCommand52); - CPPUNIT_TEST(testRemoveCommand52); - - // 6.0 tests - CPPUNIT_TEST(testPutCommandWithBucketSpace6_0); - CPPUNIT_TEST(testCreateVisitorWithBucketSpace6_0); - CPPUNIT_TEST(testRequestBucketInfoWithBucketSpace6_0); - - CPPUNIT_TEST(serialized_size_is_used_to_set_approx_size_of_storage_message); - - CPPUNIT_TEST_SUITE_END(); }; -CPPUNIT_TEST_SUITE_REGISTRATION(StorageProtocolTest); - -std::vector<std::string> StorageProtocolTest::_nonVerboseMessageStrings; -std::vector<std::string> StorageProtocolTest::_verboseMessageStrings; -std::vector<char> StorageProtocolTest::_serialization50; - -void -StorageProtocolTest::recordOutput(const api::StorageMessage& msg) -{ - std::ostringstream ost; - ost << " "; - msg.print(ost, false, " "); - _nonVerboseMessageStrings.push_back(ost.str()); - ost.str(""); - ost << " "; - msg.print(ost, true, " "); - _verboseMessageStrings.push_back(ost.str()); -} +StorageProtocolTest::~StorageProtocolTest() = default; namespace { - bool debug = false; - - struct ScopedName { - std::string _name; +std::string version_as_gtest_string(TestParamInfo<vespalib::Version> info) { + std::ostringstream ss; + auto& p = info.param; + // Dots are not allowed in test names, so convert to underscores. + ss << p.getMajor() << '_' << p.getMinor() << '_' << p.getMicro(); + return ss.str(); +} - ScopedName(const std::string& s) : _name(s) { - if (debug) std::cerr << "Starting test " << _name << "\n"; - } - ~ScopedName() { - if (debug) std::cerr << "Finished test " << _name << "\n"; - } - }; +} -} // Anonymous namespace +// TODO replace with INSTANTIATE_TEST_SUITE_P on newer gtest versions +INSTANTIATE_TEST_CASE_P(MultiVersionTest, StorageProtocolTest, + Values(vespalib::Version(6, 240, 0), + vespalib::Version(7, 40, 5)), + version_as_gtest_string); namespace { mbus::Message::UP lastCommand; mbus::Reply::UP lastReply; } -void -StorageProtocolTest::testAddress50() -{ +TEST_F(StorageProtocolTest, testAddress50) { StorageMessageAddress address("foo", lib::NodeType::STORAGE, 3); - CPPUNIT_ASSERT_EQUAL(vespalib::string("storage/cluster.foo/storage/3/default"), + EXPECT_EQ(vespalib::string("storage/cluster.foo/storage/3/default"), address.getRoute().toString()); } template<typename Command> std::shared_ptr<Command> -StorageProtocolTest::copyCommand(const std::shared_ptr<Command>& m, vespalib::Version version) +StorageProtocolTest::copyCommand(const std::shared_ptr<Command>& m) { - mbus::Message::UP mbusMessage(new mbusprot::StorageCommand(m)); + auto mbusMessage = std::make_unique<mbusprot::StorageCommand>(m); + auto version = GetParam(); mbus::Blob blob = _protocol.encode(version, *mbusMessage); mbus::Routable::UP copy(_protocol.decode(version, blob)); + assert(copy.get()); - CPPUNIT_ASSERT(copy.get()); - - mbusprot::StorageCommand* copy2(dynamic_cast<mbusprot::StorageCommand*>(copy.get())); - CPPUNIT_ASSERT(copy2 != 0); + auto* copy2 = dynamic_cast<mbusprot::StorageCommand*>(copy.get()); + assert(copy2 != nullptr); StorageCommand::SP internalMessage(copy2->getCommand()); lastCommand = std::move(mbusMessage); @@ -219,80 +143,61 @@ StorageProtocolTest::copyCommand(const std::shared_ptr<Command>& m, vespalib::Ve template<typename Reply> std::shared_ptr<Reply> StorageProtocolTest::copyReply(const std::shared_ptr<Reply>& m) { - mbus::Reply::UP mbusMessage(new mbusprot::StorageReply(m)); - mbus::Blob blob = _protocol.encode(_version5_1, *mbusMessage); - mbus::Routable::UP copy(_protocol.decode(_version5_1, blob)); - CPPUNIT_ASSERT(copy.get()); - mbusprot::StorageReply* copy2( - dynamic_cast<mbusprot::StorageReply*>(copy.get())); - CPPUNIT_ASSERT(copy2 != 0); + auto mbusMessage = std::make_unique<mbusprot::StorageReply>(m); + auto version = GetParam(); + mbus::Blob blob = _protocol.encode(version, *mbusMessage); + mbus::Routable::UP copy(_protocol.decode(version, blob)); + assert(copy.get()); + + auto* copy2 = dynamic_cast<mbusprot::StorageReply*>(copy.get()); + assert(copy2 != nullptr); + copy2->setMessage(std::move(lastCommand)); - StorageReply::SP internalMessage(copy2->getReply()); + auto internalMessage = copy2->getReply(); lastReply = std::move(mbusMessage); lastCommand = copy2->getMessage(); return std::dynamic_pointer_cast<Reply>(internalMessage); } -void -StorageProtocolTest::recordSerialization50() -{ - assert(lastCommand.get()); - assert(lastReply.get()); - for (uint32_t j=0; j<2; ++j) { - mbusprot::StorageMessage& msg(j == 0 - ? dynamic_cast<mbusprot::StorageMessage&>(*lastCommand) - : dynamic_cast<mbusprot::StorageMessage&>(*lastReply)); - msg.getInternalMessage()->forceMsgId(0); - mbus::Routable& routable(j == 0 - ? dynamic_cast<mbus::Routable&>(*lastCommand) - : dynamic_cast<mbus::Routable&>(*lastReply)); - mbus::Blob blob = _protocol.encode(_version5_0, routable); - _serialization50.push_back('\n'); - std::string type(msg.getInternalMessage()->getType().toString()); - for (uint32_t i=0, n=type.size(); i<n; ++i) { - _serialization50.push_back(type[i]); - } - _serialization50.push_back('\n'); - - for (uint32_t i=0, n=blob.size(); i<n; ++i) { - _serialization50.push_back(blob.data()[i]); - } - } -} - -void -StorageProtocolTest::testPut51() -{ - ScopedName test("testPut51"); - PutCommand::SP cmd(new PutCommand(_bucket, _testDoc, 14)); +TEST_P(StorageProtocolTest, put) { + auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); cmd->setUpdateTimestamp(Timestamp(13)); cmd->setLoadType(_loadTypes["foo"]); - PutCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(*_testDoc, *cmd2->getDocument()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("foo"), cmd2->getLoadType().getName()); - CPPUNIT_ASSERT_EQUAL(Timestamp(14), cmd2->getTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(13), cmd2->getUpdateTimestamp()); - - PutReply::SP reply(new PutReply(*cmd2)); - CPPUNIT_ASSERT(reply->hasDocument()); - CPPUNIT_ASSERT_EQUAL(*_testDoc, *reply->getDocument()); - PutReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT(reply2->hasDocument()); - CPPUNIT_ASSERT_EQUAL(*_testDoc, *reply->getDocument()); - CPPUNIT_ASSERT_EQUAL(_testDoc->getId(), reply2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(14), reply2->getTimestamp()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testUpdate51() -{ - ScopedName test("testUpdate51"); - document::DocumentUpdate::SP update(new document::DocumentUpdate(_docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId())); - std::shared_ptr<document::AssignValueUpdate> assignUpdate(new document::AssignValueUpdate(document::IntFieldValue(17))); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(*_testDoc, *cmd2->getDocument()); + EXPECT_EQ(vespalib::string("foo"), cmd2->getLoadType().getName()); + EXPECT_EQ(Timestamp(14), cmd2->getTimestamp()); + EXPECT_EQ(Timestamp(13), cmd2->getUpdateTimestamp()); + + auto reply = std::make_shared<PutReply>(*cmd2); + ASSERT_TRUE(reply->hasDocument()); + EXPECT_EQ(*_testDoc, *reply->getDocument()); + set_dummy_bucket_info_reply_fields(*reply); + auto reply2 = copyReply(reply); + ASSERT_TRUE(reply2->hasDocument()); + EXPECT_EQ(*_testDoc, *reply->getDocument()); + EXPECT_EQ(_testDoc->getId(), reply2->getDocumentId()); + EXPECT_EQ(Timestamp(14), reply2->getTimestamp()); + EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); +} + +TEST_P(StorageProtocolTest, response_without_remapped_bucket_preserves_original_bucket) { + auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); + auto cmd2 = copyCommand(cmd); + auto reply = std::make_shared<PutReply>(*cmd2); + auto reply2 = copyReply(reply); + + EXPECT_FALSE(reply2->hasBeenRemapped()); + EXPECT_EQ(_bucket_id, reply2->getBucketId()); + EXPECT_EQ(document::BucketId(), reply2->getOriginalBucketId()); + +} + +TEST_P(StorageProtocolTest, update) { + auto update = std::make_shared<document::DocumentUpdate>( + _docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId()); + auto assignUpdate = std::make_shared<document::AssignValueUpdate>(document::IntFieldValue(17)); document::FieldUpdate fieldUpdate(_testDoc->getField("headerval")); fieldUpdate.addUpdate(*assignUpdate); update->addUpdate(fieldUpdate); @@ -300,217 +205,157 @@ StorageProtocolTest::testUpdate51() update->addFieldPathUpdate(document::FieldPathUpdate::CP( new document::RemoveFieldPathUpdate("headerval", "testdoctype1.headerval > 0"))); - UpdateCommand::SP cmd(new UpdateCommand(_bucket, update, 14)); - CPPUNIT_ASSERT_EQUAL(Timestamp(0), cmd->getOldTimestamp()); + auto cmd = std::make_shared<UpdateCommand>(_bucket, update, 14); + EXPECT_EQ(Timestamp(0), cmd->getOldTimestamp()); cmd->setOldTimestamp(10); - UpdateCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(_testDocId, cmd2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(14), cmd2->getTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(10), cmd2->getOldTimestamp()); - CPPUNIT_ASSERT_EQUAL(*update, *cmd2->getUpdate()); - - UpdateReply::SP reply(new UpdateReply(*cmd2, 8)); - UpdateReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT_EQUAL(_testDocId, reply2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(14), reply2->getTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(8), reply->getOldTimestamp()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testGet51() -{ - ScopedName test("testGet51"); - GetCommand::SP cmd(new GetCommand(_bucket, _testDocId, "foo,bar,vekterli", 123)); - GetCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(_testDocId, cmd2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(123), cmd2->getBeforeTimestamp()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("foo,bar,vekterli"), cmd2->getFieldSet()); - - GetReply::SP reply(new GetReply(*cmd2, _testDoc, 100)); - GetReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT(reply2.get()); - CPPUNIT_ASSERT(reply2->getDocument().get()); - CPPUNIT_ASSERT_EQUAL(*_testDoc, *reply2->getDocument()); - CPPUNIT_ASSERT_EQUAL(_testDoc->getId(), reply2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(123), reply2->getBeforeTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(100), reply2->getLastModifiedTimestamp()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testRemove51() -{ - ScopedName test("testRemove51"); - RemoveCommand::SP cmd(new RemoveCommand(_bucket, _testDocId, 159)); - RemoveCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(_testDocId, cmd2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(159), cmd2->getTimestamp()); - - RemoveReply::SP reply(new RemoveReply(*cmd2, 48)); - reply->setBucketInfo(BucketInfo(1,2,3,4,5, true, false, 48)); - - RemoveReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT_EQUAL(_testDocId, reply2->getDocumentId()); - CPPUNIT_ASSERT_EQUAL(Timestamp(159), reply2->getTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(48), reply2->getOldTimestamp()); - CPPUNIT_ASSERT_EQUAL(BucketInfo(1,2,3,4,5, true, false, 48), - reply2->getBucketInfo()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testRevert51() -{ - ScopedName test("testRevertCommand51"); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(_testDocId, cmd2->getDocumentId()); + EXPECT_EQ(Timestamp(14), cmd2->getTimestamp()); + EXPECT_EQ(Timestamp(10), cmd2->getOldTimestamp()); + EXPECT_EQ(*update, *cmd2->getUpdate()); + + auto reply = std::make_shared<UpdateReply>(*cmd2, 8); + set_dummy_bucket_info_reply_fields(*reply); + auto reply2 = copyReply(reply); + EXPECT_EQ(_testDocId, reply2->getDocumentId()); + EXPECT_EQ(Timestamp(14), reply2->getTimestamp()); + EXPECT_EQ(Timestamp(8), reply->getOldTimestamp()); + EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); +} + +TEST_P(StorageProtocolTest, get) { + auto cmd = std::make_shared<GetCommand>(_bucket, _testDocId, "foo,bar,vekterli", 123); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(_testDocId, cmd2->getDocumentId()); + EXPECT_EQ(Timestamp(123), cmd2->getBeforeTimestamp()); + EXPECT_EQ(vespalib::string("foo,bar,vekterli"), cmd2->getFieldSet()); + + auto reply = std::make_shared<GetReply>(*cmd2, _testDoc, 100); + set_dummy_bucket_info_reply_fields(*reply); + auto reply2 = copyReply(reply); + ASSERT_TRUE(reply2.get() != nullptr); + ASSERT_TRUE(reply2->getDocument().get() != nullptr); + EXPECT_EQ(*_testDoc, *reply2->getDocument()); + EXPECT_EQ(_testDoc->getId(), reply2->getDocumentId()); + EXPECT_EQ(Timestamp(123), reply2->getBeforeTimestamp()); + EXPECT_EQ(Timestamp(100), reply2->getLastModifiedTimestamp()); + EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); +} + +TEST_P(StorageProtocolTest, remove) { + auto cmd = std::make_shared<RemoveCommand>(_bucket, _testDocId, 159); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(_testDocId, cmd2->getDocumentId()); + EXPECT_EQ(Timestamp(159), cmd2->getTimestamp()); + + auto reply = std::make_shared<RemoveReply>(*cmd2, 48); + set_dummy_bucket_info_reply_fields(*reply); + + auto reply2 = copyReply(reply); + EXPECT_EQ(_testDocId, reply2->getDocumentId()); + EXPECT_EQ(Timestamp(159), reply2->getTimestamp()); + EXPECT_EQ(Timestamp(48), reply2->getOldTimestamp()); + EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); +} + +TEST_P(StorageProtocolTest, revert) { std::vector<Timestamp> tokens; tokens.push_back(59); - RevertCommand::SP cmd(new RevertCommand(_bucket, tokens)); - RevertCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(tokens, cmd2->getRevertTokens()); - - RevertReply::SP reply(new RevertReply(*cmd2)); - BucketInfo info(0x12345432, 101, 520); - reply->setBucketInfo(info); - RevertReply::SP reply2(copyReply(reply)); + auto cmd = std::make_shared<RevertCommand>(_bucket, tokens); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(tokens, cmd2->getRevertTokens()); - CPPUNIT_ASSERT_EQUAL(info, reply2->getBucketInfo()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + auto reply = std::make_shared<RevertReply>(*cmd2); + set_dummy_bucket_info_reply_fields(*reply); + auto reply2 = copyReply(reply); + EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); } -void -StorageProtocolTest::testRequestBucketInfo51() -{ - ScopedName test("testRequestBucketInfo51"); +TEST_P(StorageProtocolTest, request_bucket_info) { { std::vector<document::BucketId> ids; ids.push_back(document::BucketId(3)); ids.push_back(document::BucketId(7)); - RequestBucketInfoCommand::SP cmd(new RequestBucketInfoCommand(makeBucketSpace(), ids)); - RequestBucketInfoCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(ids, cmd2->getBuckets()); - CPPUNIT_ASSERT(!cmd2->hasSystemState()); - - recordOutput(*cmd2); + auto cmd = std::make_shared<RequestBucketInfoCommand>(makeBucketSpace(), ids); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(ids, cmd2->getBuckets()); + EXPECT_FALSE(cmd2->hasSystemState()); } { ClusterState state("distributor:3 .1.s:d"); - RequestBucketInfoCommand::SP cmd(new RequestBucketInfoCommand( - makeBucketSpace(), - 3, state, "14")); - RequestBucketInfoCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT(cmd2->hasSystemState()); - CPPUNIT_ASSERT_EQUAL(uint16_t(3), cmd2->getDistributor()); - CPPUNIT_ASSERT_EQUAL(state, cmd2->getSystemState()); - CPPUNIT_ASSERT_EQUAL(size_t(0), cmd2->getBuckets().size()); - - RequestBucketInfoReply::SP reply(new RequestBucketInfoReply(*cmd)); + auto cmd = std::make_shared<RequestBucketInfoCommand>(makeBucketSpace(), 3, state, "14"); + auto cmd2 = copyCommand(cmd); + ASSERT_TRUE(cmd2->hasSystemState()); + EXPECT_EQ(uint16_t(3), cmd2->getDistributor()); + EXPECT_EQ(state, cmd2->getSystemState()); + EXPECT_EQ(size_t(0), cmd2->getBuckets().size()); + + auto reply = std::make_shared<RequestBucketInfoReply>(*cmd); RequestBucketInfoReply::Entry e; e._bucketId = document::BucketId(4); const uint64_t lastMod = 0x1337cafe98765432ULL; e._info = BucketInfo(43, 24, 123, 44, 124, false, true, lastMod); reply->getBucketInfo().push_back(e); - RequestBucketInfoReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT_EQUAL(size_t(1), reply2->getBucketInfo().size()); + auto reply2 = copyReply(reply); + EXPECT_EQ(size_t(1), reply2->getBucketInfo().size()); auto& entries(reply2->getBucketInfo()); - CPPUNIT_ASSERT_EQUAL(e, entries[0]); + EXPECT_EQ(e, entries[0]); // "Last modified" not counted by operator== for some reason. Testing // separately until we can figure out if this is by design or not. - CPPUNIT_ASSERT_EQUAL(lastMod, entries[0]._info.getLastModified()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + EXPECT_EQ(lastMod, entries[0]._info.getLastModified()); } } -void -StorageProtocolTest::testNotifyBucketChange51() -{ - ScopedName test("testNotifyBucketChange51"); - BucketInfo info(2, 3, 4); - document::BucketId modifiedBucketId(20, 1000); - document::Bucket modifiedBucket(makeDocumentBucket(modifiedBucketId)); - NotifyBucketChangeCommand::SP cmd(new NotifyBucketChangeCommand( - modifiedBucket, info)); - NotifyBucketChangeCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(document::BucketId(20, 1000), - cmd2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(info, cmd2->getBucketInfo()); - - NotifyBucketChangeReply::SP reply(new NotifyBucketChangeReply(*cmd)); - NotifyBucketChangeReply::SP reply2(copyReply(reply)); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testCreateBucket51() -{ - ScopedName test("testCreateBucket51"); - document::BucketId bucketId(623); - document::Bucket bucket(makeDocumentBucket(bucketId)); +TEST_P(StorageProtocolTest, notify_bucket_change) { + auto cmd = std::make_shared<NotifyBucketChangeCommand>(_bucket, _dummy_bucket_info); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(_dummy_bucket_info, cmd2->getBucketInfo()); - CreateBucketCommand::SP cmd(new CreateBucketCommand(bucket)); - CreateBucketCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(bucketId, cmd2->getBucketId()); + auto reply = std::make_shared<NotifyBucketChangeReply>(*cmd); + auto reply2 = copyReply(reply); +} - CreateBucketReply::SP reply(new CreateBucketReply(*cmd)); - CreateBucketReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT_EQUAL(bucketId, reply2->getBucketId()); +TEST_P(StorageProtocolTest, create_bucket_without_activation) { + auto cmd = std::make_shared<CreateBucketCommand>(_bucket); + EXPECT_FALSE(cmd->getActive()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_FALSE(cmd2->getActive()); - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + auto reply = std::make_shared<CreateBucketReply>(*cmd); + set_dummy_bucket_info_reply_fields(*reply); + auto reply2 = copyReply(reply); + EXPECT_NO_FATAL_FAILURE(assert_bucket_info_reply_fields_propagated(*reply2)); } -void -StorageProtocolTest::testDeleteBucket51() -{ - ScopedName test("testDeleteBucket51"); - document::BucketId bucketId(623); - document::Bucket bucket(makeDocumentBucket(bucketId)); - - DeleteBucketCommand::SP cmd(new DeleteBucketCommand(bucket)); - BucketInfo info(0x100, 200, 300); - cmd->setBucketInfo(info); - DeleteBucketCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(bucketId, cmd2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(info, cmd2->getBucketInfo()); - - DeleteBucketReply::SP reply(new DeleteBucketReply(*cmd)); +TEST_P(StorageProtocolTest, create_bucket_propagates_activation_flag) { + auto cmd = std::make_shared<CreateBucketCommand>(_bucket); + cmd->setActive(true); + auto cmd2 = copyCommand(cmd); + EXPECT_TRUE(cmd2->getActive()); +} + +TEST_P(StorageProtocolTest, delete_bucket) { + auto cmd = std::make_shared<DeleteBucketCommand>(_bucket); + cmd->setBucketInfo(_dummy_bucket_info); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(_dummy_bucket_info, cmd2->getBucketInfo()); + + auto reply = std::make_shared<DeleteBucketReply>(*cmd); // Not set automatically by constructor reply->setBucketInfo(cmd2->getBucketInfo()); - DeleteBucketReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT_EQUAL(bucketId, reply2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(info, reply2->getBucketInfo()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + auto reply2 = copyReply(reply); + EXPECT_EQ(_bucket_id, reply2->getBucketId()); + EXPECT_EQ(_dummy_bucket_info, reply2->getBucketInfo()); } -void -StorageProtocolTest::testMergeBucket51() -{ - ScopedName test("testMergeBucket51"); - document::BucketId bucketId(623); - document::Bucket bucket(makeDocumentBucket(bucketId)); - +TEST_P(StorageProtocolTest, merge_bucket) { typedef api::MergeBucketCommand::Node Node; std::vector<Node> nodes; nodes.push_back(Node(4, false)); @@ -522,152 +367,98 @@ StorageProtocolTest::testMergeBucket51() chain.push_back(7); chain.push_back(14); - MergeBucketCommand::SP cmd( - new MergeBucketCommand(bucket, nodes, Timestamp(1234), 567, chain)); - MergeBucketCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(bucketId, cmd2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(nodes, cmd2->getNodes()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), cmd2->getMaxTimestamp()); - CPPUNIT_ASSERT_EQUAL(uint32_t(567), cmd2->getClusterStateVersion()); - CPPUNIT_ASSERT_EQUAL(chain, cmd2->getChain()); - - MergeBucketReply::SP reply(new MergeBucketReply(*cmd)); - MergeBucketReply::SP reply2(copyReply(reply)); - CPPUNIT_ASSERT_EQUAL(bucketId, reply2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(nodes, reply2->getNodes()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), reply2->getMaxTimestamp()); - CPPUNIT_ASSERT_EQUAL(uint32_t(567), reply2->getClusterStateVersion()); - CPPUNIT_ASSERT_EQUAL(chain, reply2->getChain()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testSplitBucket51() -{ - ScopedName test("testSplitBucket51"); - - document::BucketId bucketId(16, 0); - document::Bucket bucket(makeDocumentBucket(bucketId)); - SplitBucketCommand::SP cmd(new SplitBucketCommand(bucket)); - CPPUNIT_ASSERT_EQUAL(0u, (uint32_t) cmd->getMinSplitBits()); - CPPUNIT_ASSERT_EQUAL(58u, (uint32_t) cmd->getMaxSplitBits()); - CPPUNIT_ASSERT_EQUAL(std::numeric_limits<uint32_t>().max(), - cmd->getMinByteSize()); - CPPUNIT_ASSERT_EQUAL(std::numeric_limits<uint32_t>().max(), - cmd->getMinDocCount()); + auto cmd = std::make_shared<MergeBucketCommand>(_bucket, nodes, Timestamp(1234), 567, chain); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(nodes, cmd2->getNodes()); + EXPECT_EQ(Timestamp(1234), cmd2->getMaxTimestamp()); + EXPECT_EQ(uint32_t(567), cmd2->getClusterStateVersion()); + EXPECT_EQ(chain, cmd2->getChain()); + + auto reply = std::make_shared<MergeBucketReply>(*cmd); + auto reply2 = copyReply(reply); + EXPECT_EQ(_bucket_id, reply2->getBucketId()); + EXPECT_EQ(nodes, reply2->getNodes()); + EXPECT_EQ(Timestamp(1234), reply2->getMaxTimestamp()); + EXPECT_EQ(uint32_t(567), reply2->getClusterStateVersion()); + EXPECT_EQ(chain, reply2->getChain()); +} + +TEST_P(StorageProtocolTest, split_bucket) { + auto cmd = std::make_shared<SplitBucketCommand>(_bucket); + EXPECT_EQ(0u, cmd->getMinSplitBits()); + EXPECT_EQ(58u, cmd->getMaxSplitBits()); + EXPECT_EQ(std::numeric_limits<uint32_t>().max(), cmd->getMinByteSize()); + EXPECT_EQ(std::numeric_limits<uint32_t>().max(), cmd->getMinDocCount()); cmd->setMinByteSize(1000); cmd->setMinDocCount(5); cmd->setMaxSplitBits(40); cmd->setMinSplitBits(20); - SplitBucketCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(20u, (uint32_t) cmd2->getMinSplitBits()); - CPPUNIT_ASSERT_EQUAL(40u, (uint32_t) cmd2->getMaxSplitBits()); - CPPUNIT_ASSERT_EQUAL(1000u, cmd2->getMinByteSize()); - CPPUNIT_ASSERT_EQUAL(5u, cmd2->getMinDocCount()); - - SplitBucketReply::SP reply(new SplitBucketReply(*cmd2)); - reply->getSplitInfo().push_back(SplitBucketReply::Entry( - document::BucketId(17, 0), BucketInfo(100, 1000, 10000, true, true))); - reply->getSplitInfo().push_back(SplitBucketReply::Entry( - document::BucketId(17, 1), BucketInfo(101, 1001, 10001, true, true))); - SplitBucketReply::SP reply2(copyReply(reply)); - - CPPUNIT_ASSERT_EQUAL(bucketId, reply2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(size_t(2), reply2->getSplitInfo().size()); - CPPUNIT_ASSERT_EQUAL(document::BucketId(17, 0), - reply2->getSplitInfo()[0].first); - CPPUNIT_ASSERT_EQUAL(document::BucketId(17, 1), - reply2->getSplitInfo()[1].first); - CPPUNIT_ASSERT_EQUAL(BucketInfo(100, 1000, 10000, true, true), - reply2->getSplitInfo()[0].second); - CPPUNIT_ASSERT_EQUAL(BucketInfo(101, 1001, 10001, true, true), - reply2->getSplitInfo()[1].second); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testJoinBuckets51() -{ - ScopedName test("testJoinBuckets51"); - document::BucketId bucketId(16, 0); - document::Bucket bucket(makeDocumentBucket(bucketId)); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + EXPECT_EQ(20u, cmd2->getMinSplitBits()); + EXPECT_EQ(40u, cmd2->getMaxSplitBits()); + EXPECT_EQ(1000u, cmd2->getMinByteSize()); + EXPECT_EQ(5u, cmd2->getMinDocCount()); + + auto reply = std::make_shared<SplitBucketReply>(*cmd2); + reply->getSplitInfo().emplace_back(document::BucketId(17, 0), BucketInfo(100, 1000, 10000, true, true)); + reply->getSplitInfo().emplace_back(document::BucketId(17, 1), BucketInfo(101, 1001, 10001, true, true)); + auto reply2 = copyReply(reply); + + EXPECT_EQ(_bucket, reply2->getBucket()); + EXPECT_EQ(size_t(2), reply2->getSplitInfo().size()); + EXPECT_EQ(document::BucketId(17, 0), reply2->getSplitInfo()[0].first); + EXPECT_EQ(document::BucketId(17, 1), reply2->getSplitInfo()[1].first); + EXPECT_EQ(BucketInfo(100, 1000, 10000, true, true), reply2->getSplitInfo()[0].second); + EXPECT_EQ(BucketInfo(101, 1001, 10001, true, true), reply2->getSplitInfo()[1].second); +} + +TEST_P(StorageProtocolTest, join_buckets) { std::vector<document::BucketId> sources; sources.push_back(document::BucketId(17, 0)); sources.push_back(document::BucketId(17, 1)); - JoinBucketsCommand::SP cmd(new JoinBucketsCommand(bucket)); + auto cmd = std::make_shared<JoinBucketsCommand>(_bucket); cmd->getSourceBuckets() = sources; cmd->setMinJoinBits(3); - JoinBucketsCommand::SP cmd2(copyCommand(cmd, _version5_1)); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); - JoinBucketsReply::SP reply(new JoinBucketsReply(*cmd2)); + auto reply = std::make_shared<JoinBucketsReply>(*cmd2); reply->setBucketInfo(BucketInfo(3,4,5)); - JoinBucketsReply::SP reply2(copyReply(reply)); + auto reply2 = copyReply(reply); - CPPUNIT_ASSERT_EQUAL(sources, reply2->getSourceBuckets()); - CPPUNIT_ASSERT_EQUAL(3, (int)cmd2->getMinJoinBits()); - CPPUNIT_ASSERT_EQUAL(BucketInfo(3,4,5), reply2->getBucketInfo()); - CPPUNIT_ASSERT_EQUAL(bucketId, reply2->getBucketId()); - - recordOutput(*cmd2); - recordOutput(*reply2); + EXPECT_EQ(sources, reply2->getSourceBuckets()); + EXPECT_EQ(3, cmd2->getMinJoinBits()); + EXPECT_EQ(BucketInfo(3,4,5), reply2->getBucketInfo()); + EXPECT_EQ(_bucket, reply2->getBucket()); } -void -StorageProtocolTest::testDestroyVisitor51() -{ - ScopedName test("testDestroyVisitor51"); +TEST_P(StorageProtocolTest, destroy_visitor) { + auto cmd = std::make_shared<DestroyVisitorCommand>("instance"); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ("instance", cmd2->getInstanceId()); - DestroyVisitorCommand::SP cmd( - new DestroyVisitorCommand("instance")); - DestroyVisitorCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(string("instance"), cmd2->getInstanceId()); - - DestroyVisitorReply::SP reply(new DestroyVisitorReply(*cmd2)); - DestroyVisitorReply::SP reply2(copyReply(reply)); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + auto reply = std::make_shared<DestroyVisitorReply>(*cmd2); + auto reply2 = copyReply(reply); } -void -StorageProtocolTest::testRemoveLocation51() -{ - ScopedName test("testRemoveLocation51"); - document::BucketId bucketId(16, 1234); - document::Bucket bucket(makeDocumentBucket(bucketId)); - - RemoveLocationCommand::SP cmd( - new RemoveLocationCommand("id.group == \"mygroup\"", bucket)); - RemoveLocationCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(vespalib::string("id.group == \"mygroup\""), cmd2->getDocumentSelection()); - CPPUNIT_ASSERT_EQUAL(bucketId, cmd2->getBucketId()); - - RemoveLocationReply::SP reply(new RemoveLocationReply(*cmd2)); - RemoveLocationReply::SP reply2(copyReply(reply)); +TEST_P(StorageProtocolTest, remove_location) { + auto cmd = std::make_shared<RemoveLocationCommand>("id.group == \"mygroup\"", _bucket); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ("id.group == \"mygroup\"", cmd2->getDocumentSelection()); + EXPECT_EQ(_bucket, cmd2->getBucket()); - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + auto reply = std::make_shared<RemoveLocationReply>(*cmd2); + auto reply2 = copyReply(reply); } -void -StorageProtocolTest::testCreateVisitor51() -{ - ScopedName test("testCreateVisitor51"); - +TEST_P(StorageProtocolTest, create_visitor) { std::vector<document::BucketId> buckets; buckets.push_back(document::BucketId(16, 1)); buckets.push_back(document::BucketId(16, 2)); - CreateVisitorCommand::SP cmd( - new CreateVisitorCommand(makeBucketSpace(), "library", "id", "doc selection")); + auto cmd = std::make_shared<CreateVisitorCommand>(makeBucketSpace(), "library", "id", "doc selection"); cmd->setControlDestination("controldest"); cmd->setDataDestination("datadest"); cmd->setVisitorCmdId(1); @@ -681,40 +472,26 @@ StorageProtocolTest::testCreateVisitor51() cmd->setFieldSet("foo,bar,vekterli"); cmd->setVisitInconsistentBuckets(); cmd->setQueueTimeout(100); - cmd->setVisitorOrdering(document::OrderingSpecification::DESCENDING); cmd->setPriority(149); - CreateVisitorCommand::SP cmd2(copyCommand(cmd, _version5_1)); - CPPUNIT_ASSERT_EQUAL(string("library"), cmd2->getLibraryName()); - CPPUNIT_ASSERT_EQUAL(string("id"), cmd2->getInstanceId()); - CPPUNIT_ASSERT_EQUAL(string("doc selection"), - cmd2->getDocumentSelection()); - CPPUNIT_ASSERT_EQUAL(string("controldest"), - cmd2->getControlDestination()); - CPPUNIT_ASSERT_EQUAL(string("datadest"), cmd2->getDataDestination()); - CPPUNIT_ASSERT_EQUAL(api::Timestamp(123), cmd2->getFromTime()); - CPPUNIT_ASSERT_EQUAL(api::Timestamp(456), cmd2->getToTime()); - CPPUNIT_ASSERT_EQUAL(2u, cmd2->getMaximumPendingReplyCount()); - CPPUNIT_ASSERT_EQUAL(buckets, cmd2->getBuckets()); - CPPUNIT_ASSERT_EQUAL(vespalib::string("foo,bar,vekterli"), cmd2->getFieldSet()); - CPPUNIT_ASSERT(cmd2->visitInconsistentBuckets()); - CPPUNIT_ASSERT_EQUAL(document::OrderingSpecification::DESCENDING, cmd2->getVisitorOrdering()); - CPPUNIT_ASSERT_EQUAL(149, (int)cmd2->getPriority()); - - CreateVisitorReply::SP reply(new CreateVisitorReply(*cmd2)); - CreateVisitorReply::SP reply2(copyReply(reply)); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); -} - -void -StorageProtocolTest::testGetBucketDiff51() -{ - ScopedName test("testGetBucketDiff51"); - document::BucketId bucketId(623); - document::Bucket bucket(makeDocumentBucket(bucketId)); - + auto cmd2 = copyCommand(cmd); + EXPECT_EQ("library", cmd2->getLibraryName()); + EXPECT_EQ("id", cmd2->getInstanceId()); + EXPECT_EQ("doc selection", cmd2->getDocumentSelection()); + EXPECT_EQ("controldest", cmd2->getControlDestination()); + EXPECT_EQ("datadest", cmd2->getDataDestination()); + EXPECT_EQ(api::Timestamp(123), cmd2->getFromTime()); + EXPECT_EQ(api::Timestamp(456), cmd2->getToTime()); + EXPECT_EQ(2u, cmd2->getMaximumPendingReplyCount()); + EXPECT_EQ(buckets, cmd2->getBuckets()); + EXPECT_EQ("foo,bar,vekterli", cmd2->getFieldSet()); + EXPECT_TRUE(cmd2->visitInconsistentBuckets()); + EXPECT_EQ(149, cmd2->getPriority()); + + auto reply = std::make_shared<CreateVisitorReply>(*cmd2); + auto reply2 = copyReply(reply); +} + +TEST_P(StorageProtocolTest, get_bucket_diff) { std::vector<api::MergeBucketCommand::Node> nodes; nodes.push_back(4); nodes.push_back(13); @@ -727,56 +504,68 @@ StorageProtocolTest::testGetBucketDiff51() entries.back()._flags = 1; entries.back()._hasMask = 3; - CPPUNIT_ASSERT_EQUAL(std::string( - "Entry(timestamp: 123456, gid(0x313233343536373839306162), " - "hasMask: 0x3,\n" - " header size: 100, body size: 65536, flags 0x1)"), - entries.back().toString(true)); + EXPECT_EQ("Entry(timestamp: 123456, gid(0x313233343536373839306162), hasMask: 0x3,\n" + " header size: 100, body size: 65536, flags 0x1)", + entries.back().toString(true)); - GetBucketDiffCommand::SP cmd(new GetBucketDiffCommand(bucket, nodes, 1056)); + auto cmd = std::make_shared<GetBucketDiffCommand>(_bucket, nodes, 1056); cmd->getDiff() = entries; - GetBucketDiffCommand::SP cmd2(copyCommand(cmd, _version5_1)); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); + + auto reply = std::make_shared<GetBucketDiffReply>(*cmd2); + EXPECT_EQ(entries, reply->getDiff()); + auto reply2 = copyReply(reply); - GetBucketDiffReply::SP reply(new GetBucketDiffReply(*cmd2)); - CPPUNIT_ASSERT_EQUAL(entries, reply->getDiff()); - GetBucketDiffReply::SP reply2(copyReply(reply)); + EXPECT_EQ(nodes, reply2->getNodes()); + EXPECT_EQ(entries, reply2->getDiff()); + EXPECT_EQ(Timestamp(1056), reply2->getMaxTimestamp()); +} + +namespace { + +ApplyBucketDiffCommand::Entry dummy_apply_entry() { + ApplyBucketDiffCommand::Entry e; + e._docName = "my cool id"; + vespalib::string header_data = "fancy header"; + e._headerBlob.resize(header_data.size()); + memcpy(&e._headerBlob[0], header_data.data(), header_data.size()); - CPPUNIT_ASSERT_EQUAL(nodes, reply2->getNodes()); - CPPUNIT_ASSERT_EQUAL(entries, reply2->getDiff()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1056), reply2->getMaxTimestamp()); + vespalib::string body_data = "fancier body!"; + e._bodyBlob.resize(body_data.size()); + memcpy(&e._bodyBlob[0], body_data.data(), body_data.size()); - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + GetBucketDiffCommand::Entry meta; + meta._timestamp = 567890; + meta._hasMask = 0x3; + meta._flags = 0x1; + meta._headerSize = 12345; + meta._headerSize = header_data.size(); + meta._bodySize = body_data.size(); + + e._entry = meta; + return e; } -void -StorageProtocolTest::testApplyBucketDiff51() -{ - ScopedName test("testApplyBucketDiff51"); - document::BucketId bucketId(16, 623); - document::Bucket bucket(makeDocumentBucket(bucketId)); +} +TEST_P(StorageProtocolTest, apply_bucket_diff) { std::vector<api::MergeBucketCommand::Node> nodes; nodes.push_back(4); nodes.push_back(13); - std::vector<ApplyBucketDiffCommand::Entry> entries; - entries.push_back(ApplyBucketDiffCommand::Entry()); + std::vector<ApplyBucketDiffCommand::Entry> entries = {dummy_apply_entry()}; - ApplyBucketDiffCommand::SP cmd(new ApplyBucketDiffCommand(bucket, nodes, 1234)); + auto cmd = std::make_shared<ApplyBucketDiffCommand>(_bucket, nodes, 1234); cmd->getDiff() = entries; - ApplyBucketDiffCommand::SP cmd2(copyCommand(cmd, _version5_1)); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); - ApplyBucketDiffReply::SP reply(new ApplyBucketDiffReply(*cmd2)); - ApplyBucketDiffReply::SP reply2(copyReply(reply)); + auto reply = std::make_shared<ApplyBucketDiffReply>(*cmd2); + auto reply2 = copyReply(reply); - CPPUNIT_ASSERT_EQUAL(nodes, reply2->getNodes()); - CPPUNIT_ASSERT_EQUAL(entries, reply2->getDiff()); - CPPUNIT_ASSERT_EQUAL(1234u, reply2->getMaxBufferSize()); - - recordOutput(*cmd2); - recordOutput(*reply2); - recordSerialization50(); + EXPECT_EQ(nodes, reply2->getNodes()); + EXPECT_EQ(entries, reply2->getDiff()); + EXPECT_EQ(1234u, reply2->getMaxBufferSize()); } namespace { @@ -807,161 +596,97 @@ namespace { }; api::StorageReply::UP MyCommand::makeReply() { - return api::StorageReply::UP(new MyReply(*this)); + return std::make_unique<MyReply>(*this); } } -void -StorageProtocolTest::testInternalMessage() -{ - ScopedName test("testInternal51"); +TEST_P(StorageProtocolTest, internal_message) { MyCommand cmd; MyReply reply(cmd); - - recordOutput(cmd); - recordOutput(reply); + // TODO what's this even intended to test? } -void -StorageProtocolTest::testSetBucketState51() -{ - ScopedName test("testSetBucketState51"); - document::BucketId bucketId(16, 0); - document::Bucket bucket(makeDocumentBucket(bucketId)); - SetBucketStateCommand::SP cmd( - new SetBucketStateCommand(bucket, SetBucketStateCommand::ACTIVE)); - SetBucketStateCommand::SP cmd2(copyCommand(cmd, _version5_1)); +TEST_P(StorageProtocolTest, set_bucket_state_with_inactive_state) { + auto cmd = std::make_shared<SetBucketStateCommand>(_bucket, SetBucketStateCommand::INACTIVE); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(_bucket, cmd2->getBucket()); - SetBucketStateReply::SP reply(new SetBucketStateReply(*cmd2)); - SetBucketStateReply::SP reply2(copyReply(reply)); + auto reply = std::make_shared<SetBucketStateReply>(*cmd2); + auto reply2 = copyReply(reply); - CPPUNIT_ASSERT_EQUAL(SetBucketStateCommand::ACTIVE, cmd2->getState()); - CPPUNIT_ASSERT_EQUAL(bucketId, cmd2->getBucketId()); - CPPUNIT_ASSERT_EQUAL(bucketId, reply2->getBucketId()); - - recordOutput(*cmd2); - recordOutput(*reply2); + EXPECT_EQ(SetBucketStateCommand::INACTIVE, cmd2->getState()); + EXPECT_EQ(_bucket, reply2->getBucket()); } -void -StorageProtocolTest::testPutCommand52() -{ - ScopedName test("testPutCommand52"); +TEST_P(StorageProtocolTest, set_bucket_state_with_active_state) { + auto cmd = std::make_shared<SetBucketStateCommand>(_bucket, SetBucketStateCommand::ACTIVE); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(SetBucketStateCommand::ACTIVE, cmd2->getState()); +} - PutCommand::SP cmd(new PutCommand(_bucket, _testDoc, 14)); +TEST_P(StorageProtocolTest, put_command_with_condition) { + auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); cmd->setCondition(TestAndSetCondition(CONDITION_STRING)); - PutCommand::SP cmd2(copyCommand(cmd, _version5_2)); - CPPUNIT_ASSERT_EQUAL(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); } -void -StorageProtocolTest::testUpdateCommand52() -{ - ScopedName test("testUpdateCommand52"); - - document::DocumentUpdate::SP update(new document::DocumentUpdate(_docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId())); - UpdateCommand::SP cmd(new UpdateCommand(_bucket, update, 14)); +TEST_P(StorageProtocolTest, update_command_with_condition) { + auto update = std::make_shared<document::DocumentUpdate>( + _docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId()); + auto cmd = std::make_shared<UpdateCommand>(_bucket, update, 14); cmd->setCondition(TestAndSetCondition(CONDITION_STRING)); - UpdateCommand::SP cmd2(copyCommand(cmd, _version5_2)); - CPPUNIT_ASSERT_EQUAL(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); } -void -StorageProtocolTest::testRemoveCommand52() -{ - ScopedName test("testRemoveCommand52"); - - RemoveCommand::SP cmd(new RemoveCommand(_bucket, _testDocId, 159)); +TEST_P(StorageProtocolTest, remove_command_with_condition) { + auto cmd = std::make_shared<RemoveCommand>(_bucket, _testDocId, 159); cmd->setCondition(TestAndSetCondition(CONDITION_STRING)); - RemoveCommand::SP cmd2(copyCommand(cmd, _version5_2)); - CPPUNIT_ASSERT_EQUAL(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); } -void -StorageProtocolTest::testPutCommandWithBucketSpace6_0() -{ - ScopedName test("testPutCommandWithBucketSpace6_0"); - - document::Bucket bucket(document::BucketSpace(5), _bucket.getBucketId()); +TEST_P(StorageProtocolTest, put_command_with_bucket_space) { + document::Bucket bucket(document::BucketSpace(5), _bucket_id); auto cmd = std::make_shared<PutCommand>(bucket, _testDoc, 14); - auto cmd2 = copyCommand(cmd, _version6_0); - CPPUNIT_ASSERT_EQUAL(bucket, cmd2->getBucket()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(bucket, cmd2->getBucket()); } -void -StorageProtocolTest::testCreateVisitorWithBucketSpace6_0() -{ - ScopedName test("testCreateVisitorWithBucketSpace6_0"); - +TEST_P(StorageProtocolTest, create_visitor_with_bucket_space) { document::BucketSpace bucketSpace(5); auto cmd = std::make_shared<CreateVisitorCommand>(bucketSpace, "library", "id", "doc selection"); - auto cmd2 = copyCommand(cmd, _version6_0); - CPPUNIT_ASSERT_EQUAL(bucketSpace, cmd2->getBucketSpace()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(bucketSpace, cmd2->getBucketSpace()); } -void -StorageProtocolTest::testRequestBucketInfoWithBucketSpace6_0() -{ - ScopedName test("testRequestBucketInfoWithBucketSpace6_0"); - +TEST_P(StorageProtocolTest, request_bucket_info_with_bucket_space) { document::BucketSpace bucketSpace(5); std::vector<document::BucketId> ids = {document::BucketId(3)}; auto cmd = std::make_shared<RequestBucketInfoCommand>(bucketSpace, ids); - auto cmd2 = copyCommand(cmd, _version6_0); - CPPUNIT_ASSERT_EQUAL(bucketSpace, cmd2->getBucketSpace()); - CPPUNIT_ASSERT_EQUAL(ids, cmd2->getBuckets()); -} - -void -StorageProtocolTest::serialized_size_is_used_to_set_approx_size_of_storage_message() -{ - ScopedName test("serialized_size_is_used_to_set_approx_size_of_storage_message"); - - PutCommand::SP cmd(new PutCommand(_bucket, _testDoc, 14)); - CPPUNIT_ASSERT_EQUAL(50u, cmd->getApproxByteSize()); - - PutCommand::SP cmd2(copyCommand(cmd, _version6_0)); - CPPUNIT_ASSERT_EQUAL(181u, cmd2->getApproxByteSize()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(bucketSpace, cmd2->getBucketSpace()); + EXPECT_EQ(ids, cmd2->getBuckets()); } -void -StorageProtocolTest::testStringOutputs() -{ - std::cerr << "\nNon verbose output:\n"; - for (uint32_t i=0, n=_nonVerboseMessageStrings.size(); i<n; ++i) { - std::cerr << _nonVerboseMessageStrings[i] << "\n"; - } - std::cerr << "\nVerbose output:\n"; - for (uint32_t i=0, n=_verboseMessageStrings.size(); i<n; ++i) { - std::cerr << _verboseMessageStrings[i] << "\n"; - } -} +TEST_P(StorageProtocolTest, serialized_size_is_used_to_set_approx_size_of_storage_message) { + auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); + EXPECT_EQ(50u, cmd->getApproxByteSize()); -void -StorageProtocolTest::testWriteSerialization50() -{ - std::ofstream of("mbusprot/mbusprot.5.0.serialization.5.1"); - of << std::hex << std::setfill('0'); - for (uint32_t i=0, n=_serialization50.size(); i<n; ++i) { - char c = _serialization50[i]; - if (c > 126 || (c < 32 && c != 10)) { - int32_t num = static_cast<int32_t>(c); - if (num < 0) num += 256; - of << '\\' << std::setw(2) << num; - } else if (c == '\\') { - of << "\\\\"; - } else { - of << c; - } + auto cmd2 = copyCommand(cmd); + auto version = GetParam(); + if (version.getMajor() == 7) { // Protobuf-based encoding + EXPECT_EQ(158u, cmd2->getApproxByteSize()); + } else { // Legacy encoding + EXPECT_EQ(181u, cmd2->getApproxByteSize()); } - of.close(); } -} // mbusprot -} // storage +} // storage::api diff --git a/storageapi/src/vespa/storageapi/CMakeLists.txt b/storageapi/src/vespa/storageapi/CMakeLists.txt index c08dcbc2419..90eb6dd9eca 100644 --- a/storageapi/src/vespa/storageapi/CMakeLists.txt +++ b/storageapi/src/vespa/storageapi/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + vespa_add_library(storageapi SOURCES $<TARGET_OBJECTS:storageapi_message> @@ -8,3 +9,6 @@ vespa_add_library(storageapi INSTALL lib64 DEPENDS ) + +vespa_add_target_package_dependency(storageapi Protobuf) + diff --git a/storageapi/src/vespa/storageapi/mbusprot/.gitignore b/storageapi/src/vespa/storageapi/mbusprot/.gitignore index 526f91c6668..8e91fe9cab0 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/.gitignore +++ b/storageapi/src/vespa/storageapi/mbusprot/.gitignore @@ -5,3 +5,6 @@ .deps .libs Makefile +*.pb.h +*.pb.cc + diff --git a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt index d5952d7cb91..dc4e3897e49 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt +++ b/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt @@ -1,4 +1,19 @@ # Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +find_package(Protobuf REQUIRED) +PROTOBUF_GENERATE_CPP(storageapi_PROTOBUF_SRCS storageapi_PROTOBUF_HDRS + protobuf/common.proto + protobuf/feed.proto + protobuf/visiting.proto + protobuf/maintenance.proto) + +# protoc-generated files emit compiler warnings that we normally treat as errors. +# Instead of rolling our own compiler plugin we'll pragmatically disable the noise. +set_source_files_properties(${storageapi_PROTOBUF_SRCS} PROPERTIES COMPILE_FLAGS "-Wno-suggest-override -Wno-inline") +# protoc explicitly annotates methods with inline, which triggers -Werror=inline when +# the header file grows over a certain size. +set_source_files_properties(protocolserialization7.cpp PROPERTIES COMPILE_FLAGS "-Wno-inline") + vespa_add_library(storageapi_mbusprot OBJECT SOURCES storagemessage.cpp @@ -11,5 +26,7 @@ vespa_add_library(storageapi_mbusprot OBJECT protocolserialization5_1.cpp protocolserialization5_2.cpp protocolserialization6_0.cpp + protocolserialization7.cpp + ${storageapi_PROTOBUF_SRCS} DEPENDS ) diff --git a/storageapi/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h b/storageapi/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h new file mode 100644 index 00000000000..ef4a6b28749 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h @@ -0,0 +1,31 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "protocolserialization.h" + +namespace storage::mbusprot { + +/* + * Utility base class for pre-v7 (protobuf) serialization implementations. + * + * TODO remove on Vespa 8 alongside legacy serialization formats. + */ +class LegacyProtocolSerialization : public ProtocolSerialization { + const std::shared_ptr<const document::DocumentTypeRepo> _repo; +public: + explicit LegacyProtocolSerialization(const std::shared_ptr<const document::DocumentTypeRepo>& repo) + : _repo(repo) + {} + + const document::DocumentTypeRepo& getTypeRepo() const { return *_repo; } + const std::shared_ptr<const document::DocumentTypeRepo> getTypeRepoSp() const { return _repo; } + + virtual document::Bucket getBucket(document::ByteBuffer& buf) const = 0; + virtual void putBucket(const document::Bucket& bucket, vespalib::GrowableByteBuffer& buf) const = 0; + virtual document::BucketSpace getBucketSpace(document::ByteBuffer& buf) const = 0; + virtual void putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer& buf) const = 0; + virtual api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const = 0; + virtual void putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const = 0; +}; + +} // storage::mbusprot diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/common.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/common.proto new file mode 100644 index 00000000000..d641449995d --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/common.proto @@ -0,0 +1,68 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +syntax = "proto3"; + +option cc_enable_arenas = true; + +package storage.mbusprot.protobuf; + +// Note: we use a *Request/*Response naming convention rather than *Command/*Reply, +// as the former is the gRPC convention and that's where we intend to move. + +message BucketSpace { + uint64 space_id = 1; +} + +message BucketId { + fixed64 raw_id = 1; +} + +message Bucket { + uint64 space_id = 1; + fixed64 raw_bucket_id = 2; +} + +// Next tag to use: 3 +message BucketInfo { + uint64 last_modified_timestamp = 1; + fixed32 legacy_checksum = 2; + // TODO v2 checksum + uint32 doc_count = 3; + uint32 total_doc_size = 4; + uint32 meta_count = 5; + uint32 used_file_size = 6; + bool ready = 7; + bool active = 8; +} + +message GlobalId { + // 96 bits of GID data in _little_ endian. High entropy, so fixed encoding is better than varint. + // Low 64 bits as if memcpy()ed from bytes [0, 8) of the GID buffer + fixed64 lo_64 = 1; + // High 32 bits as if memcpy()ed from bytes [8, 12) of the GID buffer + fixed32 hi_32 = 2; +} + +// TODO these should ideally be gRPC headers.. +message RequestHeader { + uint64 message_id = 1; + uint32 priority = 2; // Always in range [0, 255] + uint32 source_index = 3; // Always in range [0, 65535] + fixed32 loadtype_id = 4; // It's a hash with high entropy, so fixed encoding is better than varint +} + +// TODO these should ideally be gRPC headers.. +message ResponseHeader { + // TODO this should ideally be gRPC Status... + uint32 return_code_id = 1; + bytes return_code_message = 2; // FIXME it's `bytes` since `string` will check for UTF-8... might not hold... + uint64 message_id = 3; + uint32 priority = 4; // Always in range [0, 255] +} + +message Document { + bytes payload = 1; +} + +message DocumentId { + bytes id = 1; +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto new file mode 100644 index 00000000000..58da24df836 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto @@ -0,0 +1,91 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +syntax = "proto3"; + +option cc_enable_arenas = true; + +package storage.mbusprot.protobuf; + +import "common.proto"; + +message TestAndSetCondition { + bytes selection = 1; +} + +message PutRequest { + Bucket bucket = 1; + Document document = 2; + uint64 new_timestamp = 3; + uint64 expected_old_timestamp = 4; // If zero; no expectation. + TestAndSetCondition condition = 5; +} + +message PutResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; + bool was_found = 3; +} + +message Update { + bytes payload = 1; +} + +message UpdateRequest { + Bucket bucket = 1; + Update update = 2; + uint64 new_timestamp = 3; + uint64 expected_old_timestamp = 4; // If zero; no expectation. + TestAndSetCondition condition = 5; +} + +message UpdateResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; + uint64 updated_timestamp = 3; +} + +message RemoveRequest { + Bucket bucket = 1; + bytes document_id = 2; + uint64 new_timestamp = 3; + TestAndSetCondition condition = 4; +} + +message RemoveResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; + uint64 removed_timestamp = 3; +} + +message GetRequest { + Bucket bucket = 1; + bytes document_id = 2; + bytes field_set = 3; + uint64 before_timestamp = 4; +} + +message GetResponse { + Document document = 1; + uint64 last_modified_timestamp = 2; + BucketInfo bucket_info = 3; + BucketId remapped_bucket_id = 4; +} + +message RevertRequest { + Bucket bucket = 1; + repeated uint64 revert_tokens = 2; +} + +message RevertResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; +} + +message RemoveLocationRequest { + Bucket bucket = 1; + bytes document_selection = 2; +} + +message RemoveLocationResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto new file mode 100644 index 00000000000..c4766d2900a --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto @@ -0,0 +1,160 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +syntax = "proto3"; + +option cc_enable_arenas = true; + +package storage.mbusprot.protobuf; + +import "common.proto"; + +message DeleteBucketRequest { + Bucket bucket = 1; + BucketInfo expected_bucket_info = 2; +} + +message DeleteBucketResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; +} + +message CreateBucketRequest { + Bucket bucket = 1; + bool create_as_active = 2; +} + +message CreateBucketResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; +} + +message MergeNode { + uint32 index = 1; + bool source_only = 2; +} + +message MergeBucketRequest { + Bucket bucket = 1; + uint32 cluster_state_version = 2; + uint64 max_timestamp = 3; + repeated MergeNode nodes = 4; + repeated uint32 node_chain = 5; +} + +message MergeBucketResponse { + BucketId remapped_bucket_id = 1; +} + +message MetaDiffEntry { + uint64 timestamp = 1; + GlobalId gid = 2; + uint32 header_size = 3; + uint32 body_size = 4; + uint32 flags = 5; + uint32 presence_mask = 6; +} + +message GetBucketDiffRequest { + Bucket bucket = 1; + uint64 max_timestamp = 2; + repeated MergeNode nodes = 3; + repeated MetaDiffEntry diff = 4; +} + +message GetBucketDiffResponse { + BucketId remapped_bucket_id = 1; + repeated MetaDiffEntry diff = 2; +} + +message ApplyDiffEntry { + MetaDiffEntry entry_meta = 1; + bytes document_id = 2; + bytes header_blob = 3; + bytes body_blob = 4; +} + +message ApplyBucketDiffRequest { + Bucket bucket = 1; + repeated MergeNode nodes = 2; + uint32 max_buffer_size = 3; + repeated ApplyDiffEntry entries = 4; +} + +message ApplyBucketDiffResponse { + BucketId remapped_bucket_id = 1; + repeated ApplyDiffEntry entries = 4; +} + +message ExplicitBucketSet { + // `Bucket` is not needed, as the space is inferred from the owning message. + repeated BucketId bucket_ids = 2; +} + +message AllBuckets { + uint32 distributor_index = 1; + bytes cluster_state = 2; + bytes distribution_hash = 3; +} + +message RequestBucketInfoRequest { + BucketSpace bucket_space = 1; + oneof request_for { + ExplicitBucketSet explicit_bucket_set = 2; + AllBuckets all_buckets = 3; + } +} + +message BucketAndBucketInfo { + fixed64 raw_bucket_id = 1; + BucketInfo bucket_info = 2; +} + +message RequestBucketInfoResponse { + repeated BucketAndBucketInfo bucket_infos = 1; +} + +message NotifyBucketChangeRequest { + Bucket bucket = 1; + BucketInfo bucket_info = 2; +} + +message NotifyBucketChangeResponse { + // Currently empty +} + +message SplitBucketRequest { + Bucket bucket = 1; + uint32 min_split_bits = 2; + uint32 max_split_bits = 3; + uint32 min_byte_size = 4; + uint32 min_doc_count = 5; +} + +message SplitBucketResponse { + BucketId remapped_bucket_id = 1; + repeated BucketAndBucketInfo split_info = 2; +} + +message JoinBucketsRequest { + Bucket bucket = 1; + repeated BucketId source_buckets = 2; + uint32 min_join_bits = 3; +} + +message JoinBucketsResponse { + BucketInfo bucket_info = 1; + BucketId remapped_bucket_id = 2; +} + +message SetBucketStateRequest { + enum BucketState { + Inactive = 0; + Active = 1; + } + + Bucket bucket = 1; + BucketState state = 2; +} + +message SetBucketStateResponse { + BucketId remapped_bucket_id = 1; +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/visiting.proto b/storageapi/src/vespa/storageapi/mbusprot/protobuf/visiting.proto new file mode 100644 index 00000000000..89ce39e52a0 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf/visiting.proto @@ -0,0 +1,66 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +syntax = "proto3"; + +option cc_enable_arenas = true; + +package storage.mbusprot.protobuf; + +import "common.proto"; + +message ClientVisitorParameter { + bytes key = 1; + bytes value = 2; +} + +message VisitorConstraints { + bytes document_selection = 1; + uint64 from_time_usec = 2; + uint64 to_time_usec = 3; + bool visit_removes = 4; + bytes field_set = 5; + bool visit_inconsistent_buckets = 6; +} + +message VisitorControlMeta { + bytes instance_id = 1; + bytes library_name = 2; + uint32 visitor_command_id = 3; + bytes control_destination = 4; + bytes data_destination = 5; + + // TODO move? + uint32 max_pending_reply_count = 6; + uint32 queue_timeout = 7; + uint32 max_buckets_per_visitor = 8; +} + +message CreateVisitorRequest { + BucketSpace bucket_space = 1; + repeated BucketId buckets = 2; + + VisitorConstraints constraints = 3; + VisitorControlMeta control_meta = 4; + repeated ClientVisitorParameter client_parameters = 5; +} + +message VisitorStatistics { + uint32 buckets_visited = 1; + uint64 documents_visited = 2; + uint64 bytes_visited = 3; + uint64 documents_returned = 4; + uint64 bytes_returned = 5; + uint64 second_pass_documents_returned = 6; // TODO don't include? orderdoc only + uint64 second_pass_bytes_returned = 7; // TODO don't include? orderdoc only +} + +message CreateVisitorResponse { + VisitorStatistics visitor_statistics = 1; +} + +message DestroyVisitorRequest { + bytes instance_id = 1; +} + +message DestroyVisitorResponse { + // Currently empty +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h b/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h new file mode 100644 index 00000000000..8e878cf0560 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h @@ -0,0 +1,13 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +// Disable warnings emitted by protoc generated files +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsuggest-override" + +#include "feed.pb.h" +#include "visiting.pb.h" +#include "maintenance.pb.h" + +#pragma GCC diagnostic pop diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp index 172cd6c8de5..917b60c50c3 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp @@ -17,11 +17,6 @@ LOG_SETUP(".storage.api.mbusprot.serialization.base"); namespace storage::mbusprot { -ProtocolSerialization::ProtocolSerialization(const std::shared_ptr<const document::DocumentTypeRepo>& repo) - : _repo(repo) -{ -} - mbus::Blob ProtocolSerialization::encode(const api::StorageMessage& msg) const { diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h index 9c3ddb88bdf..a57627b9ba9 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h @@ -59,21 +59,14 @@ class StorageCommand; class StorageReply; class ProtocolSerialization { - const std::shared_ptr<const document::DocumentTypeRepo> _repo; - public: virtual mbus::Blob encode(const api::StorageMessage&) const; virtual std::unique_ptr<StorageCommand> decodeCommand(mbus::BlobRef) const; virtual std::unique_ptr<StorageReply> decodeReply( mbus::BlobRef, const api::StorageCommand&) const; - protected: - const document::DocumentTypeRepo& getTypeRepo() const { return *_repo; } - const std::shared_ptr<const document::DocumentTypeRepo> getTypeRepoSp() const - { return _repo; } - - ProtocolSerialization(const std::shared_ptr<const document::DocumentTypeRepo> &repo); - virtual ~ProtocolSerialization() {} + ProtocolSerialization() = default; + virtual ~ProtocolSerialization() = default; typedef api::StorageCommand SCmd; typedef api::StorageReply SRep; @@ -102,13 +95,10 @@ protected: virtual void onEncode(GBBuf&, const api::GetBucketDiffReply&) const = 0; virtual void onEncode(GBBuf&, const api::ApplyBucketDiffCommand&) const = 0; virtual void onEncode(GBBuf&, const api::ApplyBucketDiffReply&) const = 0; - virtual void onEncode(GBBuf&, - const api::RequestBucketInfoCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::RequestBucketInfoCommand&) const = 0; virtual void onEncode(GBBuf&, const api::RequestBucketInfoReply&) const = 0; - virtual void onEncode(GBBuf&, - const api::NotifyBucketChangeCommand&) const = 0; - virtual void onEncode(GBBuf&, - const api::NotifyBucketChangeReply&) const = 0; + virtual void onEncode(GBBuf&, const api::NotifyBucketChangeCommand&) const = 0; + virtual void onEncode(GBBuf&, const api::NotifyBucketChangeReply&) const = 0; virtual void onEncode(GBBuf&, const api::SplitBucketCommand&) const = 0; virtual void onEncode(GBBuf&, const api::SplitBucketReply&) const = 0; virtual void onEncode(GBBuf&, const api::JoinBucketsCommand&) const = 0; @@ -143,11 +133,9 @@ protected: virtual SCmd::UP onDecodeApplyBucketDiffCommand(BBuf&) const = 0; virtual SRep::UP onDecodeApplyBucketDiffReply(const SCmd&, BBuf&) const = 0; virtual SCmd::UP onDecodeRequestBucketInfoCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeRequestBucketInfoReply(const SCmd&, - BBuf&) const = 0; + virtual SRep::UP onDecodeRequestBucketInfoReply(const SCmd&, BBuf&) const = 0; virtual SCmd::UP onDecodeNotifyBucketChangeCommand(BBuf&) const = 0; - virtual SRep::UP onDecodeNotifyBucketChangeReply(const SCmd&, - BBuf&) const = 0; + virtual SRep::UP onDecodeNotifyBucketChangeReply(const SCmd&, BBuf&) const = 0; virtual SCmd::UP onDecodeSplitBucketCommand(BBuf&) const = 0; virtual SRep::UP onDecodeSplitBucketReply(const SCmd&, BBuf&) const = 0; virtual SCmd::UP onDecodeJoinBucketsCommand(BBuf&) const = 0; @@ -160,14 +148,6 @@ protected: virtual SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const = 0; virtual SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const = 0; virtual SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const = 0; - - virtual document::Bucket getBucket(document::ByteBuffer& buf) const = 0; - virtual void putBucket(const document::Bucket& bucket, vespalib::GrowableByteBuffer& buf) const = 0; - virtual document::BucketSpace getBucketSpace(document::ByteBuffer& buf) const = 0; - virtual void putBucketSpace(document::BucketSpace bucketSpace, vespalib::GrowableByteBuffer& buf) const = 0; - virtual api::BucketInfo getBucketInfo(document::ByteBuffer& buf) const = 0; - virtual void putBucketInfo(const api::BucketInfo& info, vespalib::GrowableByteBuffer& buf) const = 0; - }; } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp index 74a0c964d19..466ff85f398 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp @@ -20,7 +20,7 @@ namespace storage::mbusprot { ProtocolSerialization4_2::ProtocolSerialization4_2( const std::shared_ptr<const document::DocumentTypeRepo>& repo) - : ProtocolSerialization(repo) + : LegacyProtocolSerialization(repo) { } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h index 56aa3d4ed30..e4ab36dc989 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h @@ -1,13 +1,13 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "protocolserialization.h" +#include "legacyprotocolserialization.h" namespace storage::mbusprot { -class ProtocolSerialization4_2 : public ProtocolSerialization { +class ProtocolSerialization4_2 : public LegacyProtocolSerialization { public: - ProtocolSerialization4_2(const std::shared_ptr<const document::DocumentTypeRepo>&); + explicit ProtocolSerialization4_2(const std::shared_ptr<const document::DocumentTypeRepo>&); protected: void onEncode(GBBuf&, const api::GetCommand&) const override; diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h index 042ec7850ef..67f02aa2d2a 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h @@ -73,6 +73,9 @@ public: SRep::UP onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) const override; void onDecodeCommand(BBuf& buf, api::StorageCommand& msg) const override; void onDecodeReply(BBuf&, api::StorageReply&) const override; + +protected: + const documentapi::LoadTypeSet& loadTypes() const noexcept { return _loadTypes; }; }; } diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp new file mode 100644 index 00000000000..ca77977046c --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -0,0 +1,1268 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "protocolserialization7.h" +#include "serializationhelper.h" +#include "protobuf_includes.h" + +#include <vespa/document/update/documentupdate.h> +#include <vespa/document/util/bufferexceptions.h> +#include <vespa/storageapi/message/bucketsplitting.h> +#include <vespa/storageapi/message/persistence.h> +#include <vespa/storageapi/message/removelocation.h> +#include <vespa/storageapi/message/visitor.h> + +namespace storage::mbusprot { + +ProtocolSerialization7::ProtocolSerialization7(std::shared_ptr<const document::DocumentTypeRepo> repo, + const documentapi::LoadTypeSet& load_types) + : ProtocolSerialization(), + _repo(std::move(repo)), + _load_types(load_types) +{ +} + +namespace { + +void set_bucket(protobuf::Bucket& dest, const document::Bucket& src) { + dest.set_raw_bucket_id(src.getBucketId().getRawId()); + dest.set_space_id(src.getBucketSpace().getId()); +} + +void set_bucket_id(protobuf::BucketId& dest, const document::BucketId& src) { + dest.set_raw_id(src.getRawId()); +} + +document::BucketId get_bucket_id(const protobuf::BucketId& src) { + return document::BucketId(src.raw_id()); +} + +void set_bucket_space(protobuf::BucketSpace& dest, const document::BucketSpace& src) { + dest.set_space_id(src.getId()); +} + +document::BucketSpace get_bucket_space(const protobuf::BucketSpace& src) { + return document::BucketSpace(src.space_id()); +} + +void set_bucket_info(protobuf::BucketInfo& dest, const api::BucketInfo& src) { + dest.set_last_modified_timestamp(src.getLastModified()); + dest.set_legacy_checksum(src.getChecksum()); + dest.set_doc_count(src.getDocumentCount()); + dest.set_total_doc_size(src.getTotalDocumentSize()); + dest.set_meta_count(src.getMetaCount()); + dest.set_used_file_size(src.getUsedFileSize()); + dest.set_active(src.isActive()); + dest.set_ready(src.isReady()); +} + +document::Bucket get_bucket(const protobuf::Bucket& src) { + return document::Bucket(document::BucketSpace(src.space_id()), + document::BucketId(src.raw_bucket_id())); +} + +api::BucketInfo get_bucket_info(const protobuf::BucketInfo& src) { + api::BucketInfo info; + info.setLastModified(src.last_modified_timestamp()); + info.setChecksum(src.legacy_checksum()); + info.setDocumentCount(src.doc_count()); + info.setTotalDocumentSize(src.total_doc_size()); + info.setMetaCount(src.meta_count()); + info.setUsedFileSize(src.used_file_size()); + info.setActive(src.active()); + info.setReady(src.ready()); + return info; +} + +documentapi::TestAndSetCondition get_tas_condition(const protobuf::TestAndSetCondition& src) { + return documentapi::TestAndSetCondition(src.selection()); +} + +void set_tas_condition(protobuf::TestAndSetCondition& dest, const documentapi::TestAndSetCondition& src) { + dest.set_selection(src.getSelection().data(), src.getSelection().size()); +} + +std::shared_ptr<document::Document> get_document(const protobuf::Document& src_doc, + const document::DocumentTypeRepo& type_repo) +{ + if (!src_doc.payload().empty()) { + document::ByteBuffer doc_buf(src_doc.payload().data(), src_doc.payload().size()); + return std::make_shared<document::Document>(type_repo, doc_buf); + } + return std::shared_ptr<document::Document>(); +} + +void set_update(protobuf::Update& dest, const document::DocumentUpdate& src) { + vespalib::nbostream stream; + src.serializeHEAD(stream); + dest.set_payload(stream.peek(), stream.size()); +} + +std::shared_ptr<document::DocumentUpdate> get_update(const protobuf::Update& src, + const document::DocumentTypeRepo& type_repo) +{ + if (!src.payload().empty()) { + return document::DocumentUpdate::createHEAD( + type_repo, vespalib::nbostream(src.payload().data(), src.payload().size())); + } + return std::shared_ptr<document::DocumentUpdate>(); +} + +void write_request_header(vespalib::GrowableByteBuffer& buf, const api::StorageCommand& cmd) { + protobuf::RequestHeader hdr; // Arena alloc not needed since there are no nested messages + hdr.set_message_id(cmd.getMsgId()); + hdr.set_priority(cmd.getPriority()); + hdr.set_source_index(cmd.getSourceIndex()); + hdr.set_loadtype_id(cmd.getLoadType().getId()); + + uint8_t dest[128]; // Only primitive fields, should be plenty large enough. + auto encoded_size = static_cast<uint32_t>(hdr.ByteSizeLong()); + assert(encoded_size <= sizeof(dest)); + [[maybe_unused]] bool ok = hdr.SerializeWithCachedSizesToArray(dest); + assert(ok); + buf.putInt(encoded_size); + buf.putBytes(reinterpret_cast<const char*>(dest), encoded_size); +} + +void write_response_header(vespalib::GrowableByteBuffer& buf, const api::StorageReply& reply) { + protobuf::ResponseHeader hdr; // Arena alloc not needed since there are no nested messages + const auto& result = reply.getResult(); + hdr.set_return_code_id(static_cast<uint32_t>(result.getResult())); + if (!result.getMessage().empty()) { + hdr.set_return_code_message(result.getMessage().data(), result.getMessage().size()); + } + hdr.set_message_id(reply.getMsgId()); + hdr.set_priority(reply.getPriority()); + + const auto header_size = hdr.ByteSizeLong(); + assert(header_size <= UINT32_MAX); + buf.putInt(static_cast<uint32_t>(header_size)); + + auto* dest_buf = reinterpret_cast<uint8_t*>(buf.allocate(header_size)); + [[maybe_unused]] bool ok = hdr.SerializeWithCachedSizesToArray(dest_buf); + assert(ok); +} + +void decode_request_header(document::ByteBuffer& buf, protobuf::RequestHeader& hdr) { + auto hdr_len = static_cast<uint32_t>(SerializationHelper::getInt(buf)); + if (hdr_len > buf.getRemaining()) { + throw document::BufferOutOfBoundsException(buf.getPos(), hdr_len); + } + bool ok = hdr.ParseFromArray(buf.getBufferAtPos(), hdr_len); + if (!ok) { + throw vespalib::IllegalArgumentException("Malformed protobuf request header"); + } + buf.incPos(hdr_len); +} + +void decode_response_header(document::ByteBuffer& buf, protobuf::ResponseHeader& hdr) { + auto hdr_len = static_cast<uint32_t>(SerializationHelper::getInt(buf)); + if (hdr_len > buf.getRemaining()) { + throw document::BufferOutOfBoundsException(buf.getPos(), hdr_len); + } + bool ok = hdr.ParseFromArray(buf.getBufferAtPos(), hdr_len); + if (!ok) { + throw vespalib::IllegalArgumentException("Malformed protobuf response header"); + } + buf.incPos(hdr_len); +} + +} // anonymous namespace + +template <typename ProtobufType> +class BaseEncoder { + vespalib::GrowableByteBuffer& _out_buf; + ::google::protobuf::Arena _arena; + ProtobufType* _proto_obj; +public: + explicit BaseEncoder(vespalib::GrowableByteBuffer& out_buf) + : _out_buf(out_buf), + _arena(), + _proto_obj(::google::protobuf::Arena::Create<ProtobufType>(&_arena)) + { + } + + void encode() { + assert(_proto_obj != nullptr); + const auto sz = _proto_obj->ByteSizeLong(); + assert(sz <= UINT32_MAX); + auto* buf = reinterpret_cast<uint8_t*>(_out_buf.allocate(sz)); + [[maybe_unused]] bool ok = _proto_obj->SerializeWithCachedSizesToArray(buf); + assert(ok); + _proto_obj = nullptr; + } +protected: + vespalib::GrowableByteBuffer& buffer() noexcept { return _out_buf; } + + // Precondition: encode() is not called + ProtobufType& proto_obj() noexcept { return *_proto_obj; } + const ProtobufType& proto_obj() const noexcept { return *_proto_obj; } +}; + +template <typename ProtobufType> +class RequestEncoder : public BaseEncoder<ProtobufType> { +public: + RequestEncoder(vespalib::GrowableByteBuffer& out_buf, const api::StorageCommand& cmd) + : BaseEncoder<ProtobufType>(out_buf) + { + write_request_header(out_buf, cmd); + } + + // Precondition: encode() is not called + ProtobufType& request() noexcept { return this->proto_obj(); } + const ProtobufType& request() const noexcept { return this->proto_obj(); } +}; + +template <typename ProtobufType> +class ResponseEncoder : public BaseEncoder<ProtobufType> { +public: + ResponseEncoder(vespalib::GrowableByteBuffer& out_buf, const api::StorageReply& reply) + : BaseEncoder<ProtobufType>(out_buf) + { + write_response_header(out_buf, reply); + } + + // Precondition: encode() is not called + ProtobufType& response() noexcept { return this->proto_obj(); } + const ProtobufType& response() const noexcept { return this->proto_obj(); } +}; + +template <typename ProtobufType> +class RequestDecoder { + protobuf::RequestHeader _hdr; + ::google::protobuf::Arena _arena; + ProtobufType* _proto_obj; + const documentapi::LoadTypeSet& _load_types; +public: + RequestDecoder(document::ByteBuffer& in_buf, const documentapi::LoadTypeSet& load_types) + : _arena(), + _proto_obj(::google::protobuf::Arena::Create<ProtobufType>(&_arena)), + _load_types(load_types) + { + decode_request_header(in_buf, _hdr); + assert(in_buf.getRemaining() <= INT_MAX); + bool ok = _proto_obj->ParseFromArray(in_buf.getBufferAtPos(), in_buf.getRemaining()); + if (!ok) { + throw vespalib::IllegalArgumentException( + vespalib::make_string("Malformed protobuf request payload for %s", + ProtobufType::descriptor()->full_name().c_str())); + } + } + + void transfer_meta_information_to(api::StorageCommand& dest) { + dest.forceMsgId(_hdr.message_id()); + dest.setPriority(static_cast<uint8_t>(_hdr.priority())); + dest.setSourceIndex(static_cast<uint16_t>(_hdr.source_index())); + dest.setLoadType(_load_types[_hdr.loadtype_id()]); + } + + ProtobufType& request() noexcept { return *_proto_obj; } + const ProtobufType& request() const noexcept { return *_proto_obj; } +}; + +template <typename ProtobufType> +class ResponseDecoder { + protobuf::ResponseHeader _hdr; + ::google::protobuf::Arena _arena; + ProtobufType* _proto_obj; +public: + explicit ResponseDecoder(document::ByteBuffer& in_buf) + : _arena(), + _proto_obj(::google::protobuf::Arena::Create<ProtobufType>(&_arena)) + { + decode_response_header(in_buf, _hdr); + assert(in_buf.getRemaining() <= INT_MAX); + bool ok = _proto_obj->ParseFromArray(in_buf.getBufferAtPos(), in_buf.getRemaining()); + if (!ok) { + throw vespalib::IllegalArgumentException( + vespalib::make_string("Malformed protobuf response payload for %s", + ProtobufType::descriptor()->full_name().c_str())); + } + } + + ProtobufType& response() noexcept { return *_proto_obj; } + const ProtobufType& response() const noexcept { return *_proto_obj; } +}; + +template <typename ProtobufType, typename Func> +void encode_request(vespalib::GrowableByteBuffer& out_buf, const api::StorageCommand& msg, Func&& f) { + RequestEncoder<ProtobufType> enc(out_buf, msg); + f(enc.request()); + enc.encode(); +} + +template <typename ProtobufType, typename Func> +void encode_response(vespalib::GrowableByteBuffer& out_buf, const api::StorageReply& reply, Func&& f) { + ResponseEncoder<ProtobufType> enc(out_buf, reply); + auto& res = enc.response(); + f(res); + enc.encode(); +} + +template <typename ProtobufType, typename Func> +std::unique_ptr<api::StorageCommand> +ProtocolSerialization7::decode_request(document::ByteBuffer& in_buf, Func&& f) const { + RequestDecoder<ProtobufType> dec(in_buf, _load_types); + const auto& req = dec.request(); + auto cmd = f(req); + dec.transfer_meta_information_to(*cmd); + return cmd; +} + +template <typename ProtobufType, typename Func> +std::unique_ptr<api::StorageReply> +ProtocolSerialization7::decode_response(document::ByteBuffer& in_buf, Func&& f) const { + ResponseDecoder<ProtobufType> dec(in_buf); + const auto& res = dec.response(); + auto reply = f(res); + return reply; +} + +template <typename ProtobufType, typename Func> +void encode_bucket_request(vespalib::GrowableByteBuffer& out_buf, const api::BucketCommand& msg, Func&& f) { + encode_request<ProtobufType>(out_buf, msg, [&](ProtobufType& req) { + set_bucket(*req.mutable_bucket(), msg.getBucket()); + f(req); + }); +} + +template <typename ProtobufType, typename Func> +std::unique_ptr<api::StorageCommand> +ProtocolSerialization7::decode_bucket_request(document::ByteBuffer& in_buf, Func&& f) const { + return decode_request<ProtobufType>(in_buf, [&](const ProtobufType& req) { + if (!req.has_bucket()) { + throw vespalib::IllegalArgumentException( + vespalib::make_string("Malformed protocol buffer request for %s; no bucket", + ProtobufType::descriptor()->full_name().c_str())); + } + const auto bucket = get_bucket(req.bucket()); + return f(req, bucket); + }); +} + +template <typename ProtobufType, typename Func> +void encode_bucket_response(vespalib::GrowableByteBuffer& out_buf, const api::BucketReply& reply, Func&& f) { + encode_response<ProtobufType>(out_buf, reply, [&](ProtobufType& res) { + if (reply.hasBeenRemapped()) { + set_bucket_id(*res.mutable_remapped_bucket_id(), reply.getBucketId()); + } + f(res); + }); +} + +template <typename ProtobufType, typename Func> +std::unique_ptr<api::StorageReply> +ProtocolSerialization7::decode_bucket_response(document::ByteBuffer& in_buf, Func&& f) const { + return decode_response<ProtobufType>(in_buf, [&](const ProtobufType& res) { + auto reply = f(res); + if (res.has_remapped_bucket_id()) { + reply->remapBucketId(get_bucket_id(res.remapped_bucket_id())); + } + return reply; + }); +} + +template <typename ProtobufType, typename Func> +void encode_bucket_info_response(vespalib::GrowableByteBuffer& out_buf, const api::BucketInfoReply& reply, Func&& f) { + encode_bucket_response<ProtobufType>(out_buf, reply, [&](ProtobufType& res) { + set_bucket_info(*res.mutable_bucket_info(), reply.getBucketInfo()); + f(res); + }); +} + +template <typename ProtobufType, typename Func> +std::unique_ptr<api::StorageReply> +ProtocolSerialization7::decode_bucket_info_response(document::ByteBuffer& in_buf, Func&& f) const { + return decode_bucket_response<ProtobufType>(in_buf, [&](const ProtobufType& res) { + auto reply = f(res); + if (res.has_bucket_info()) { + reply->setBucketInfo(get_bucket_info(res.bucket_info())); + } + return reply; + }); +} + +// TODO document protobuf ducktyping assumptions + +namespace { +// Inherit from known base class just to avoid having to template this. We don't care about its subtype anyway. +void no_op_encode([[maybe_unused]] ::google::protobuf::Message&) { + // nothing to do here. +} + +void set_document(protobuf::Document& target_doc, const document::Document& src_doc) { + vespalib::nbostream stream; + src_doc.serialize(stream); + target_doc.set_payload(stream.peek(), stream.size()); +} + +} + +// ----------------------------------------------------------------- +// Put +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::PutCommand& msg) const { + encode_bucket_request<protobuf::PutRequest>(buf, msg, [&](auto& req) { + req.set_new_timestamp(msg.getTimestamp()); + req.set_expected_old_timestamp(msg.getUpdateTimestamp()); + if (msg.getCondition().isPresent()) { + set_tas_condition(*req.mutable_condition(), msg.getCondition()); + } + if (msg.getDocument()) { + set_document(*req.mutable_document(), *msg.getDocument()); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::PutReply& msg) const { + encode_bucket_info_response<protobuf::PutResponse>(buf, msg, [&](auto& res) { + res.set_was_found(msg.wasFound()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodePutCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::PutRequest>(buf, [&](auto& req, auto& bucket) { + auto document = get_document(req.document(), type_repo()); + auto cmd = std::make_unique<api::PutCommand>(bucket, std::move(document), req.new_timestamp()); + cmd->setUpdateTimestamp(req.expected_old_timestamp()); + if (req.has_condition()) { + cmd->setCondition(get_tas_condition(req.condition())); + } + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodePutReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::PutResponse>(buf, [&](auto& res) { + return std::make_unique<api::PutReply>(static_cast<const api::PutCommand&>(cmd), res.was_found()); + }); +} + +// ----------------------------------------------------------------- +// Update +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::UpdateCommand& msg) const { + encode_bucket_request<protobuf::UpdateRequest>(buf, msg, [&](auto& req) { + auto* update = msg.getUpdate().get(); + if (update) { + set_update(*req.mutable_update(), *update); + } + req.set_new_timestamp(msg.getTimestamp()); + req.set_expected_old_timestamp(msg.getOldTimestamp()); + if (msg.getCondition().isPresent()) { + set_tas_condition(*req.mutable_condition(), msg.getCondition()); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::UpdateReply& msg) const { + encode_bucket_info_response<protobuf::UpdateResponse>(buf, msg, [&](auto& res) { + res.set_updated_timestamp(msg.getOldTimestamp()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeUpdateCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::UpdateRequest>(buf, [&](auto& req, auto& bucket) { + auto update = get_update(req.update(), type_repo()); + auto cmd = std::make_unique<api::UpdateCommand>(bucket, std::move(update), req.new_timestamp()); + cmd->setOldTimestamp(req.expected_old_timestamp()); + if (req.has_condition()) { + cmd->setCondition(get_tas_condition(req.condition())); + } + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeUpdateReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::UpdateResponse>(buf, [&](auto& res) { + return std::make_unique<api::UpdateReply>(static_cast<const api::UpdateCommand&>(cmd), + res.updated_timestamp()); + }); +} + +// ----------------------------------------------------------------- +// Remove +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveCommand& msg) const { + encode_bucket_request<protobuf::RemoveRequest>(buf, msg, [&](auto& req) { + auto doc_id_str = msg.getDocumentId().toString(); + req.set_document_id(doc_id_str.data(), doc_id_str.size()); + req.set_new_timestamp(msg.getTimestamp()); + if (msg.getCondition().isPresent()) { + set_tas_condition(*req.mutable_condition(), msg.getCondition()); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveReply& msg) const { + encode_bucket_info_response<protobuf::RemoveResponse>(buf, msg, [&](auto& res) { + res.set_removed_timestamp(msg.getOldTimestamp()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::RemoveRequest>(buf, [&](auto& req, auto& bucket) { + document::DocumentId doc_id(vespalib::stringref(req.document_id().data(), req.document_id().size())); + auto cmd = std::make_unique<api::RemoveCommand>(bucket, doc_id, req.new_timestamp()); + if (req.has_condition()) { + cmd->setCondition(get_tas_condition(req.condition())); + } + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeRemoveReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::RemoveResponse>(buf, [&](auto& res) { + return std::make_unique<api::RemoveReply>(static_cast<const api::RemoveCommand&>(cmd), + res.removed_timestamp()); + }); +} + +// ----------------------------------------------------------------- +// Get +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::GetCommand& msg) const { + encode_bucket_request<protobuf::GetRequest>(buf, msg, [&](auto& req) { + auto doc_id = msg.getDocumentId().toString(); + req.set_document_id(doc_id.data(), doc_id.size()); + req.set_before_timestamp(msg.getBeforeTimestamp()); + if (!msg.getFieldSet().empty()) { + req.set_field_set(msg.getFieldSet().data(), msg.getFieldSet().size()); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::GetReply& msg) const { + encode_bucket_info_response<protobuf::GetResponse>(buf, msg, [&](auto& res) { + if (msg.getDocument()) { + set_document(*res.mutable_document(), *msg.getDocument()); + } + res.set_last_modified_timestamp(msg.getLastModifiedTimestamp()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeGetCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::GetRequest>(buf, [&](auto& req, auto& bucket) { + document::DocumentId doc_id(vespalib::stringref(req.document_id().data(), req.document_id().size())); + return std::make_unique<api::GetCommand>(bucket, std::move(doc_id), + req.field_set(), req.before_timestamp()); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeGetReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::GetResponse>(buf, [&](auto& res) { + try { + auto document = get_document(res.document(), type_repo()); + return std::make_unique<api::GetReply>(static_cast<const api::GetCommand&>(cmd), + std::move(document), res.last_modified_timestamp()); + } catch (std::exception& e) { + auto reply = std::make_unique<api::GetReply>(static_cast<const api::GetCommand&>(cmd), + std::shared_ptr<document::Document>(), 0u); + reply->setResult(api::ReturnCode(api::ReturnCode::UNPARSEABLE, e.what())); + return reply; + } + }); +} + +// ----------------------------------------------------------------- +// Revert +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RevertCommand& msg) const { + encode_bucket_request<protobuf::RevertRequest>(buf, msg, [&](auto& req) { + auto* tokens = req.mutable_revert_tokens(); + assert(msg.getRevertTokens().size() <= INT_MAX); + tokens->Reserve(static_cast<int>(msg.getRevertTokens().size())); + for (auto token : msg.getRevertTokens()) { + tokens->Add(token); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RevertReply& msg) const { + encode_bucket_info_response<protobuf::RevertResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeRevertCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::RevertRequest>(buf, [&](auto& req, auto& bucket) { + std::vector<api::Timestamp> tokens; + tokens.reserve(req.revert_tokens_size()); + for (auto token : req.revert_tokens()) { + tokens.emplace_back(api::Timestamp(token)); + } + return std::make_unique<api::RevertCommand>(bucket, std::move(tokens)); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeRevertReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::RevertResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::RevertReply>(static_cast<const api::RevertCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// RemoveLocation +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationCommand& msg) const { + encode_bucket_request<protobuf::RemoveLocationRequest>(buf, msg, [&](auto& req) { + req.set_document_selection(msg.getDocumentSelection().data(), msg.getDocumentSelection().size()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RemoveLocationReply& msg) const { + encode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeRemoveLocationCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::RemoveLocationRequest>(buf, [&](auto& req, auto& bucket) { + return std::make_unique<api::RemoveLocationCommand>(req.document_selection(), bucket); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeRemoveLocationReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::RemoveLocationResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::RemoveLocationReply>(static_cast<const api::RemoveLocationCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// DeleteBucket +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::DeleteBucketCommand& msg) const { + encode_bucket_request<protobuf::DeleteBucketRequest>(buf, msg, [&](auto& req) { + set_bucket_info(*req.mutable_expected_bucket_info(), msg.getBucketInfo()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::DeleteBucketReply& msg) const { + encode_bucket_info_response<protobuf::DeleteBucketResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeDeleteBucketCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::DeleteBucketRequest>(buf, [&](auto& req, auto& bucket) { + auto cmd = std::make_unique<api::DeleteBucketCommand>(bucket); + if (req.has_expected_bucket_info()) { + cmd->setBucketInfo(get_bucket_info(req.expected_bucket_info())); + } + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeDeleteBucketReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::DeleteBucketResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::DeleteBucketReply>(static_cast<const api::DeleteBucketCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// CreateBucket +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateBucketCommand& msg) const { + encode_bucket_request<protobuf::CreateBucketRequest>(buf, msg, [&](auto& req) { + req.set_create_as_active(msg.getActive()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateBucketReply& msg) const { + encode_bucket_info_response<protobuf::CreateBucketResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeCreateBucketCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::CreateBucketRequest>(buf, [&](auto& req, auto& bucket) { + auto cmd = std::make_unique<api::CreateBucketCommand>(bucket); + cmd->setActive(req.create_as_active()); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeCreateBucketReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::CreateBucketResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::CreateBucketReply>(static_cast<const api::CreateBucketCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// MergeBucket +// ----------------------------------------------------------------- + +namespace { + +void set_merge_nodes(::google::protobuf::RepeatedPtrField<protobuf::MergeNode>& dest, + const std::vector<api::MergeBucketCommand::Node>& src) +{ + dest.Reserve(src.size()); + for (const auto& src_node : src) { + auto* dest_node = dest.Add(); + dest_node->set_index(src_node.index); + dest_node->set_source_only(src_node.sourceOnly); + } +} + +std::vector<api::MergeBucketCommand::Node> get_merge_nodes( + const ::google::protobuf::RepeatedPtrField<protobuf::MergeNode>& src) +{ + std::vector<api::MergeBucketCommand::Node> nodes; + nodes.reserve(src.size()); + for (const auto& node : src) { + nodes.emplace_back(node.index(), node.source_only()); + } + return nodes; +} + +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::MergeBucketCommand& msg) const { + encode_bucket_request<protobuf::MergeBucketRequest>(buf, msg, [&](auto& req) { + set_merge_nodes(*req.mutable_nodes(), msg.getNodes()); + req.set_max_timestamp(msg.getMaxTimestamp()); + req.set_cluster_state_version(msg.getClusterStateVersion()); + for (uint16_t chain_node : msg.getChain()) { + req.add_node_chain(chain_node); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::MergeBucketReply& msg) const { + encode_bucket_response<protobuf::MergeBucketResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeMergeBucketCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::MergeBucketRequest>(buf, [&](auto& req, auto& bucket) { + auto nodes = get_merge_nodes(req.nodes()); + auto cmd = std::make_unique<api::MergeBucketCommand>(bucket, std::move(nodes), req.max_timestamp()); + cmd->setClusterStateVersion(req.cluster_state_version()); + std::vector<uint16_t> chain; + chain.reserve(req.node_chain_size()); + for (uint16_t node : req.node_chain()) { + chain.emplace_back(node); + } + cmd->setChain(std::move(chain)); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeMergeBucketReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_response<protobuf::MergeBucketResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::MergeBucketReply>(static_cast<const api::MergeBucketCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// GetBucketDiff +// ----------------------------------------------------------------- + +namespace { + +void set_global_id(protobuf::GlobalId& dest, const document::GlobalId& src) { + static_assert(document::GlobalId::LENGTH == 12); + uint64_t lo64; + uint32_t hi32; + memcpy(&lo64, src.get(), sizeof(uint64_t)); + memcpy(&hi32, src.get() + sizeof(uint64_t), sizeof(uint32_t)); + dest.set_lo_64(lo64); + dest.set_hi_32(hi32); +} + +document::GlobalId get_global_id(const protobuf::GlobalId& src) { + static_assert(document::GlobalId::LENGTH == 12); + const uint64_t lo64 = src.lo_64(); + const uint32_t hi32 = src.hi_32(); + + char buf[document::GlobalId::LENGTH]; + memcpy(buf, &lo64, sizeof(uint64_t)); + memcpy(buf + sizeof(uint64_t), &hi32, sizeof(uint32_t)); + return document::GlobalId(buf); +} + +void set_diff_entry(protobuf::MetaDiffEntry& dest, const api::GetBucketDiffCommand::Entry& src) { + dest.set_timestamp(src._timestamp); + set_global_id(*dest.mutable_gid(), src._gid); + dest.set_header_size(src._headerSize); + dest.set_body_size(src._bodySize); + dest.set_flags(src._flags); + dest.set_presence_mask(src._hasMask); +} + +api::GetBucketDiffCommand::Entry get_diff_entry(const protobuf::MetaDiffEntry& src) { + api::GetBucketDiffCommand::Entry e; + e._timestamp = src.timestamp(); + e._gid = get_global_id(src.gid()); + e._headerSize = src.header_size(); + e._bodySize = src.body_size(); + e._flags = src.flags(); + e._hasMask = src.presence_mask(); + return e; +} + +void fill_proto_meta_diff(::google::protobuf::RepeatedPtrField<protobuf::MetaDiffEntry>& dest, + const std::vector<api::GetBucketDiffCommand::Entry>& src) { + for (const auto& diff_entry : src) { + set_diff_entry(*dest.Add(), diff_entry); + } +} + +void fill_api_meta_diff(std::vector<api::GetBucketDiffCommand::Entry>& dest, + const ::google::protobuf::RepeatedPtrField<protobuf::MetaDiffEntry>& src) { + // FIXME GetBucketDiffReply ctor copies the diff from the request for some reason + // TODO verify this isn't actually used anywhere and remove this "feature". + dest.clear(); + dest.reserve(src.size()); + for (const auto& diff_entry : src) { + dest.emplace_back(get_diff_entry(diff_entry)); + } +} + +} // anonymous namespace + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::GetBucketDiffCommand& msg) const { + encode_bucket_request<protobuf::GetBucketDiffRequest>(buf, msg, [&](auto& req) { + set_merge_nodes(*req.mutable_nodes(), msg.getNodes()); + req.set_max_timestamp(msg.getMaxTimestamp()); + fill_proto_meta_diff(*req.mutable_diff(), msg.getDiff()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::GetBucketDiffReply& msg) const { + encode_bucket_response<protobuf::GetBucketDiffResponse>(buf, msg, [&](auto& res) { + fill_proto_meta_diff(*res.mutable_diff(), msg.getDiff()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeGetBucketDiffCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::GetBucketDiffRequest>(buf, [&](auto& req, auto& bucket) { + auto nodes = get_merge_nodes(req.nodes()); + auto cmd = std::make_unique<api::GetBucketDiffCommand>(bucket, std::move(nodes), req.max_timestamp()); + fill_api_meta_diff(cmd->getDiff(), req.diff()); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeGetBucketDiffReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_response<protobuf::GetBucketDiffResponse>(buf, [&](auto& res) { + auto reply = std::make_unique<api::GetBucketDiffReply>(static_cast<const api::GetBucketDiffCommand&>(cmd)); + fill_api_meta_diff(reply->getDiff(), res.diff()); + return reply; + }); +} + +// ----------------------------------------------------------------- +// ApplyBucketDiff +// ----------------------------------------------------------------- + +namespace { + +void fill_api_apply_diff_vector(std::vector<api::ApplyBucketDiffCommand::Entry>& diff, + const ::google::protobuf::RepeatedPtrField<protobuf::ApplyDiffEntry>& src) +{ + // We use the same approach as the legacy protocols here in that we pre-reserve and + // directly write into the vector. This avoids having to ensure all buffer management is movable. + size_t n_entries = src.size(); + diff.resize(n_entries); + for (size_t i = 0; i < n_entries; ++i) { + auto& proto_entry = src.Get(i); + auto& dest = diff[i]; + dest._entry = get_diff_entry(proto_entry.entry_meta()); + dest._docName = proto_entry.document_id(); + // TODO consider making buffers std::strings instead to avoid explicit zeroing-on-resize overhead + dest._headerBlob.resize(proto_entry.header_blob().size()); + memcpy(dest._headerBlob.data(), proto_entry.header_blob().data(), proto_entry.header_blob().size()); + dest._bodyBlob.resize(proto_entry.body_blob().size()); + memcpy(dest._bodyBlob.data(), proto_entry.body_blob().data(), proto_entry.body_blob().size()); + } +} + +void fill_proto_apply_diff_vector(::google::protobuf::RepeatedPtrField<protobuf::ApplyDiffEntry>& dest, + const std::vector<api::ApplyBucketDiffCommand::Entry>& src) +{ + dest.Reserve(src.size()); + for (const auto& entry : src) { + auto* proto_entry = dest.Add(); + set_diff_entry(*proto_entry->mutable_entry_meta(), entry._entry); + proto_entry->set_document_id(entry._docName.data(), entry._docName.size()); + proto_entry->set_header_blob(entry._headerBlob.data(), entry._headerBlob.size()); + proto_entry->set_body_blob(entry._bodyBlob.data(), entry._bodyBlob.size()); + } +} + +} // anonymous namespace + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::ApplyBucketDiffCommand& msg) const { + encode_bucket_request<protobuf::ApplyBucketDiffRequest>(buf, msg, [&](auto& req) { + set_merge_nodes(*req.mutable_nodes(), msg.getNodes()); + req.set_max_buffer_size(msg.getMaxBufferSize()); + fill_proto_apply_diff_vector(*req.mutable_entries(), msg.getDiff()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::ApplyBucketDiffReply& msg) const { + encode_bucket_response<protobuf::ApplyBucketDiffResponse>(buf, msg, [&](auto& res) { + fill_proto_apply_diff_vector(*res.mutable_entries(), msg.getDiff()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeApplyBucketDiffCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::ApplyBucketDiffRequest>(buf, [&](auto& req, auto& bucket) { + auto nodes = get_merge_nodes(req.nodes()); + auto cmd = std::make_unique<api::ApplyBucketDiffCommand>(bucket, std::move(nodes), req.max_buffer_size()); + fill_api_apply_diff_vector(cmd->getDiff(), req.entries()); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeApplyBucketDiffReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_response<protobuf::ApplyBucketDiffResponse>(buf, [&](auto& res) { + auto reply = std::make_unique<api::ApplyBucketDiffReply>(static_cast<const api::ApplyBucketDiffCommand&>(cmd)); + fill_api_apply_diff_vector(reply->getDiff(), res.entries()); + return reply; + }); +} + +// ----------------------------------------------------------------- +// RequestBucketInfo +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RequestBucketInfoCommand& msg) const { + encode_request<protobuf::RequestBucketInfoRequest>(buf, msg, [&](auto& req) { + set_bucket_space(*req.mutable_bucket_space(), msg.getBucketSpace()); + auto& buckets = msg.getBuckets(); + if (!buckets.empty()) { + auto* proto_buckets = req.mutable_explicit_bucket_set(); + for (const auto& b : buckets) { + set_bucket_id(*proto_buckets->add_bucket_ids(), b); + } + } else { + auto* all_buckets = req.mutable_all_buckets(); + auto cluster_state = msg.getSystemState().toString(); + all_buckets->set_distributor_index(msg.getDistributor()); + all_buckets->set_cluster_state(cluster_state.data(), cluster_state.size()); + all_buckets->set_distribution_hash(msg.getDistributionHash().data(), msg.getDistributionHash().size()); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::RequestBucketInfoReply& msg) const { + encode_response<protobuf::RequestBucketInfoResponse>(buf, msg, [&](auto& res) { + auto* proto_info = res.mutable_bucket_infos(); + proto_info->Reserve(msg.getBucketInfo().size()); + for (const auto& entry : msg.getBucketInfo()) { + auto* bucket_and_info = proto_info->Add(); + bucket_and_info->set_raw_bucket_id(entry._bucketId.getRawId()); + set_bucket_info(*bucket_and_info->mutable_bucket_info(), entry._info); + } + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeRequestBucketInfoCommand(BBuf& buf) const { + return decode_request<protobuf::RequestBucketInfoRequest>(buf, [&](auto& req) { + auto bucket_space = get_bucket_space(req.bucket_space()); + if (req.has_explicit_bucket_set()) { + const uint32_t n_buckets = req.explicit_bucket_set().bucket_ids_size(); + std::vector<document::BucketId> buckets(n_buckets); + const auto& proto_buckets = req.explicit_bucket_set().bucket_ids(); + for (uint32_t i = 0; i < n_buckets; ++i) { + buckets[i] = get_bucket_id(proto_buckets.Get(i)); + } + return std::make_unique<api::RequestBucketInfoCommand>(bucket_space, std::move(buckets)); + } else if (req.has_all_buckets()) { + const auto& all_req = req.all_buckets(); + return std::make_unique<api::RequestBucketInfoCommand>( + bucket_space, all_req.distributor_index(), + lib::ClusterState(all_req.cluster_state()), all_req.distribution_hash()); + } else { + throw vespalib::IllegalArgumentException("RequestBucketInfo does not have any applicable fields set"); + } + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeRequestBucketInfoReply(const SCmd& cmd, BBuf& buf) const { + return decode_response<protobuf::RequestBucketInfoResponse>(buf, [&](auto& res) { + auto reply = std::make_unique<api::RequestBucketInfoReply>(static_cast<const api::RequestBucketInfoCommand&>(cmd)); + auto& dest_entries = reply->getBucketInfo(); + uint32_t n_entries = res.bucket_infos_size(); + dest_entries.resize(n_entries); + for (uint32_t i = 0; i < n_entries; ++i) { + const auto& proto_entry = res.bucket_infos(i); + dest_entries[i]._bucketId = document::BucketId(proto_entry.raw_bucket_id()); + dest_entries[i]._info = get_bucket_info(proto_entry.bucket_info()); + } + return reply; + }); +} + +// ----------------------------------------------------------------- +// NotifyBucketChange +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::NotifyBucketChangeCommand& msg) const { + encode_bucket_request<protobuf::NotifyBucketChangeRequest>(buf, msg, [&](auto& req) { + set_bucket_info(*req.mutable_bucket_info(), msg.getBucketInfo()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::NotifyBucketChangeReply& msg) const { + encode_response<protobuf::NotifyBucketChangeResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeNotifyBucketChangeCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::NotifyBucketChangeRequest>(buf, [&](auto& req, auto& bucket) { + auto bucket_info = get_bucket_info(req.bucket_info()); + return std::make_unique<api::NotifyBucketChangeCommand>(bucket, bucket_info); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeNotifyBucketChangeReply(const SCmd& cmd, BBuf& buf) const { + return decode_response<protobuf::NotifyBucketChangeResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::NotifyBucketChangeReply>(static_cast<const api::NotifyBucketChangeCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// SplitBucket +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::SplitBucketCommand& msg) const { + encode_bucket_request<protobuf::SplitBucketRequest>(buf, msg, [&](auto& req) { + req.set_min_split_bits(msg.getMinSplitBits()); + req.set_max_split_bits(msg.getMaxSplitBits()); + req.set_min_byte_size(msg.getMinByteSize()); + req.set_min_doc_count(msg.getMinDocCount()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::SplitBucketReply& msg) const { + encode_bucket_response<protobuf::SplitBucketResponse>(buf, msg, [&](auto& res) { + for (const auto& split_info : msg.getSplitInfo()) { + auto* proto_info = res.add_split_info(); + proto_info->set_raw_bucket_id(split_info.first.getRawId()); + set_bucket_info(*proto_info->mutable_bucket_info(), split_info.second); + } + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeSplitBucketCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::SplitBucketRequest>(buf, [&](auto& req, auto& bucket) { + auto cmd = std::make_unique<api::SplitBucketCommand>(bucket); + cmd->setMinSplitBits(static_cast<uint8_t>(req.min_split_bits())); + cmd->setMaxSplitBits(static_cast<uint8_t>(req.max_split_bits())); + cmd->setMinByteSize(req.min_byte_size()); + cmd->setMinDocCount(req.min_doc_count()); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeSplitBucketReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_response<protobuf::SplitBucketResponse>(buf, [&](auto& res) { + auto reply = std::make_unique<api::SplitBucketReply>(static_cast<const api::SplitBucketCommand&>(cmd)); + auto& dest_info = reply->getSplitInfo(); + dest_info.reserve(res.split_info_size()); + for (const auto& proto_info : res.split_info()) { + dest_info.emplace_back(document::BucketId(proto_info.raw_bucket_id()), + get_bucket_info(proto_info.bucket_info())); + } + return reply; + }); +} + +// ----------------------------------------------------------------- +// JoinBuckets +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::JoinBucketsCommand& msg) const { + encode_bucket_request<protobuf::JoinBucketsRequest>(buf, msg, [&](auto& req) { + for (const auto& source : msg.getSourceBuckets()) { + set_bucket_id(*req.add_source_buckets(), source); + } + req.set_min_join_bits(msg.getMinJoinBits()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::JoinBucketsReply& msg) const { + encode_bucket_info_response<protobuf::JoinBucketsResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeJoinBucketsCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::JoinBucketsRequest>(buf, [&](auto& req, auto& bucket) { + auto cmd = std::make_unique<api::JoinBucketsCommand>(bucket); + auto& entries = cmd->getSourceBuckets(); + for (const auto& proto_bucket : req.source_buckets()) { + entries.emplace_back(get_bucket_id(proto_bucket)); + } + cmd->setMinJoinBits(static_cast<uint8_t>(req.min_join_bits())); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeJoinBucketsReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_info_response<protobuf::JoinBucketsResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::JoinBucketsReply>(static_cast<const api::JoinBucketsCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// SetBucketState +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::SetBucketStateCommand& msg) const { + encode_bucket_request<protobuf::SetBucketStateRequest>(buf, msg, [&](auto& req) { + auto state = (msg.getState() == api::SetBucketStateCommand::BUCKET_STATE::ACTIVE + ? protobuf::SetBucketStateRequest_BucketState_Active + : protobuf::SetBucketStateRequest_BucketState_Inactive); + req.set_state(state); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::SetBucketStateReply& msg) const { + // SetBucketStateReply is _technically_ a BucketInfoReply, but the legacy protocol impls + // do _not_ encode bucket info as part of the wire format (and it's not used on the distributor), + // so we follow that here and only encode remapping information. + encode_bucket_response<protobuf::SetBucketStateResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeSetBucketStateCommand(BBuf& buf) const { + return decode_bucket_request<protobuf::SetBucketStateRequest>(buf, [&](auto& req, auto& bucket) { + auto state = (req.state() == protobuf::SetBucketStateRequest_BucketState_Active + ? api::SetBucketStateCommand::BUCKET_STATE::ACTIVE + : api::SetBucketStateCommand::BUCKET_STATE::INACTIVE); + return std::make_unique<api::SetBucketStateCommand>(bucket, state); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeSetBucketStateReply(const SCmd& cmd, BBuf& buf) const { + return decode_bucket_response<protobuf::SetBucketStateResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::SetBucketStateReply>(static_cast<const api::SetBucketStateCommand&>(cmd)); + }); +} + +// ----------------------------------------------------------------- +// CreateVisitor +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateVisitorCommand& msg) const { + encode_request<protobuf::CreateVisitorRequest>(buf, msg, [&](auto& req) { + set_bucket_space(*req.mutable_bucket_space(), msg.getBucketSpace()); + for (const auto& bucket : msg.getBuckets()) { + set_bucket_id(*req.add_buckets(), bucket); + } + + auto* ctrl_meta = req.mutable_control_meta(); + ctrl_meta->set_library_name(msg.getLibraryName().data(), msg.getLibraryName().size()); + ctrl_meta->set_instance_id(msg.getInstanceId().data(), msg.getInstanceId().size()); + ctrl_meta->set_visitor_command_id(msg.getVisitorCmdId()); + ctrl_meta->set_control_destination(msg.getControlDestination().data(), msg.getControlDestination().size()); + ctrl_meta->set_data_destination(msg.getDataDestination().data(), msg.getDataDestination().size()); + ctrl_meta->set_queue_timeout(msg.getQueueTimeout()); + ctrl_meta->set_max_pending_reply_count(msg.getMaximumPendingReplyCount()); + ctrl_meta->set_max_buckets_per_visitor(msg.getMaxBucketsPerVisitor()); + + auto* constraints = req.mutable_constraints(); + constraints->set_document_selection(msg.getDocumentSelection().data(), msg.getDocumentSelection().size()); + constraints->set_from_time_usec(msg.getFromTime()); + constraints->set_to_time_usec(msg.getToTime()); + constraints->set_visit_inconsistent_buckets(msg.visitInconsistentBuckets()); + constraints->set_visit_removes(msg.visitRemoves()); + constraints->set_field_set(msg.getFieldSet().data(), msg.getFieldSet().size()); + + for (const auto& param : msg.getParameters()) { + auto* proto_param = req.add_client_parameters(); + proto_param->set_key(param.first.data(), param.first.size()); + proto_param->set_value(param.second.data(), param.second.size()); + } + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::CreateVisitorReply& msg) const { + encode_response<protobuf::CreateVisitorResponse>(buf, msg, [&](auto& res) { + auto& stats = msg.getVisitorStatistics(); + auto* proto_stats = res.mutable_visitor_statistics(); + proto_stats->set_buckets_visited(stats.getBucketsVisited()); + proto_stats->set_documents_visited(stats.getDocumentsVisited()); + proto_stats->set_bytes_visited(stats.getBytesVisited()); + proto_stats->set_documents_returned(stats.getDocumentsReturned()); + proto_stats->set_bytes_returned(stats.getBytesReturned()); + proto_stats->set_second_pass_documents_returned(stats.getSecondPassDocumentsReturned()); + proto_stats->set_second_pass_bytes_returned(stats.getSecondPassBytesReturned()); + }); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeCreateVisitorCommand(BBuf& buf) const { + return decode_request<protobuf::CreateVisitorRequest>(buf, [&](auto& req) { + auto bucket_space = get_bucket_space(req.bucket_space()); + auto& ctrl_meta = req.control_meta(); + auto& constraints = req.constraints(); + auto cmd = std::make_unique<api::CreateVisitorCommand>(bucket_space, ctrl_meta.library_name(), + ctrl_meta.instance_id(), constraints.document_selection()); + for (const auto& proto_bucket : req.buckets()) { + cmd->getBuckets().emplace_back(get_bucket_id(proto_bucket)); + } + + cmd->setVisitorCmdId(ctrl_meta.visitor_command_id()); + cmd->setControlDestination(ctrl_meta.control_destination()); + cmd->setDataDestination(ctrl_meta.data_destination()); + cmd->setMaximumPendingReplyCount(ctrl_meta.max_pending_reply_count()); + cmd->setQueueTimeout(ctrl_meta.queue_timeout()); + cmd->setMaxBucketsPerVisitor(ctrl_meta.max_buckets_per_visitor()); + cmd->setVisitorDispatcherVersion(50); // FIXME this magic number is lifted verbatim from the 5.1 protocol impl + + for (const auto& proto_param : req.client_parameters()) { + cmd->getParameters().set(proto_param.key(), proto_param.value()); + } + + cmd->setFromTime(constraints.from_time_usec()); + cmd->setToTime(constraints.to_time_usec()); + cmd->setVisitRemoves(constraints.visit_removes()); + cmd->setFieldSet(constraints.field_set()); + cmd->setVisitInconsistentBuckets(constraints.visit_inconsistent_buckets()); + return cmd; + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeCreateVisitorReply(const SCmd& cmd, BBuf& buf) const { + return decode_response<protobuf::CreateVisitorResponse>(buf, [&](auto& res) { + auto reply = std::make_unique<api::CreateVisitorReply>(static_cast<const api::CreateVisitorCommand&>(cmd)); + vdslib::VisitorStatistics vs; + const auto& proto_stats = res.visitor_statistics(); + vs.setBucketsVisited(proto_stats.buckets_visited()); + vs.setDocumentsVisited(proto_stats.documents_visited()); + vs.setBytesVisited(proto_stats.bytes_visited()); + vs.setDocumentsReturned(proto_stats.documents_returned()); + vs.setBytesReturned(proto_stats.bytes_returned()); + vs.setSecondPassDocumentsReturned(proto_stats.second_pass_documents_returned()); + vs.setSecondPassBytesReturned(proto_stats.second_pass_bytes_returned()); + reply->setVisitorStatistics(vs); + return reply; + }); +} + +// ----------------------------------------------------------------- +// DestroyVisitor +// ----------------------------------------------------------------- + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::DestroyVisitorCommand& msg) const { + encode_request<protobuf::DestroyVisitorRequest>(buf, msg, [&](auto& req) { + req.set_instance_id(msg.getInstanceId().data(), msg.getInstanceId().size()); + }); +} + +void ProtocolSerialization7::onEncode(GBBuf& buf, const api::DestroyVisitorReply& msg) const { + encode_response<protobuf::DestroyVisitorResponse>(buf, msg, no_op_encode); +} + +api::StorageCommand::UP ProtocolSerialization7::onDecodeDestroyVisitorCommand(BBuf& buf) const { + return decode_request<protobuf::DestroyVisitorRequest>(buf, [&](auto& req) { + return std::make_unique<api::DestroyVisitorCommand>(req.instance_id()); + }); +} + +api::StorageReply::UP ProtocolSerialization7::onDecodeDestroyVisitorReply(const SCmd& cmd, BBuf& buf) const { + return decode_response<protobuf::DestroyVisitorResponse>(buf, [&]([[maybe_unused]] auto& res) { + return std::make_unique<api::DestroyVisitorReply>(static_cast<const api::DestroyVisitorCommand&>(cmd)); + }); +} + +} // storage::mbusprot diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h new file mode 100644 index 00000000000..f3499150278 --- /dev/null +++ b/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h @@ -0,0 +1,146 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "protocolserialization.h" +#include <vespa/documentapi/loadtypes/loadtypeset.h> + +namespace storage { +namespace mbusprot { + +/** + * Protocol serialization version that uses Protocol Buffers for all its binary + * encoding and decoding. + */ +class ProtocolSerialization7 : public ProtocolSerialization { + const std::shared_ptr<const document::DocumentTypeRepo> _repo; + const documentapi::LoadTypeSet& _load_types; +public: + ProtocolSerialization7(std::shared_ptr<const document::DocumentTypeRepo> repo, + const documentapi::LoadTypeSet& load_types); + + const document::DocumentTypeRepo& type_repo() const { return *_repo; } + + // Put + void onEncode(GBBuf&, const api::PutCommand&) const override; + void onEncode(GBBuf&, const api::PutReply&) const override; + SCmd::UP onDecodePutCommand(BBuf&) const override; + SRep::UP onDecodePutReply(const SCmd&, BBuf&) const override; + + // Update + void onEncode(GBBuf&, const api::UpdateCommand&) const override; + void onEncode(GBBuf&, const api::UpdateReply&) const override; + SCmd::UP onDecodeUpdateCommand(BBuf&) const override; + SRep::UP onDecodeUpdateReply(const SCmd&, BBuf&) const override; + + // Remove + void onEncode(GBBuf&, const api::RemoveCommand&) const override; + void onEncode(GBBuf&, const api::RemoveReply&) const override; + SCmd::UP onDecodeRemoveCommand(BBuf&) const override; + SRep::UP onDecodeRemoveReply(const SCmd&, BBuf&) const override; + + // Get + void onEncode(GBBuf&, const api::GetCommand&) const override; + void onEncode(GBBuf&, const api::GetReply&) const override; + SCmd::UP onDecodeGetCommand(BBuf&) const override; + SRep::UP onDecodeGetReply(const SCmd&, BBuf&) const override; + + // Revert - TODO this is deprecated, no? + void onEncode(GBBuf&, const api::RevertCommand&) const override; + void onEncode(GBBuf&, const api::RevertReply&) const override; + SCmd::UP onDecodeRevertCommand(BBuf&) const override; + SRep::UP onDecodeRevertReply(const SCmd&, BBuf&) const override; + + // DeleteBucket + void onEncode(GBBuf&, const api::DeleteBucketCommand&) const override; + void onEncode(GBBuf&, const api::DeleteBucketReply&) const override; + SCmd::UP onDecodeDeleteBucketCommand(BBuf&) const override; + SRep::UP onDecodeDeleteBucketReply(const SCmd&, BBuf&) const override; + + // CreateBucket + void onEncode(GBBuf&, const api::CreateBucketCommand&) const override; + void onEncode(GBBuf&, const api::CreateBucketReply&) const override; + SCmd::UP onDecodeCreateBucketCommand(BBuf&) const override; + SRep::UP onDecodeCreateBucketReply(const SCmd&, BBuf&) const override; + + // MergeBucket + void onEncode(GBBuf&, const api::MergeBucketCommand&) const override; + void onEncode(GBBuf&, const api::MergeBucketReply&) const override; + SCmd::UP onDecodeMergeBucketCommand(BBuf&) const override; + SRep::UP onDecodeMergeBucketReply(const SCmd&, BBuf&) const override; + + // GetBucketDiff + void onEncode(GBBuf&, const api::GetBucketDiffCommand&) const override; + void onEncode(GBBuf&, const api::GetBucketDiffReply&) const override; + SCmd::UP onDecodeGetBucketDiffCommand(BBuf&) const override; + SRep::UP onDecodeGetBucketDiffReply(const SCmd&, BBuf&) const override; + + // ApplyBucketDiff + void onEncode(GBBuf&, const api::ApplyBucketDiffCommand&) const override; + void onEncode(GBBuf&, const api::ApplyBucketDiffReply&) const override; + SCmd::UP onDecodeApplyBucketDiffCommand(BBuf&) const override; + SRep::UP onDecodeApplyBucketDiffReply(const SCmd&, BBuf&) const override; + + // RequestBucketInfo + void onEncode(GBBuf&, const api::RequestBucketInfoCommand&) const override; + void onEncode(GBBuf&, const api::RequestBucketInfoReply&) const override; + SCmd::UP onDecodeRequestBucketInfoCommand(BBuf&) const override; + SRep::UP onDecodeRequestBucketInfoReply(const SCmd&, BBuf&) const override; + + // NotifyBucketChange + void onEncode(GBBuf&, const api::NotifyBucketChangeCommand&) const override; + void onEncode(GBBuf&, const api::NotifyBucketChangeReply&) const override; + SCmd::UP onDecodeNotifyBucketChangeCommand(BBuf&) const override; + SRep::UP onDecodeNotifyBucketChangeReply(const SCmd&, BBuf&) const override; + + // SplitBucket + void onEncode(GBBuf&, const api::SplitBucketCommand&) const override; + void onEncode(GBBuf&, const api::SplitBucketReply&) const override; + SCmd::UP onDecodeSplitBucketCommand(BBuf&) const override; + SRep::UP onDecodeSplitBucketReply(const SCmd&, BBuf&) const override; + + // JoinBuckets + void onEncode(GBBuf&, const api::JoinBucketsCommand&) const override; + void onEncode(GBBuf&, const api::JoinBucketsReply&) const override; + SCmd::UP onDecodeJoinBucketsCommand(BBuf&) const override; + SRep::UP onDecodeJoinBucketsReply(const SCmd&, BBuf&) const override; + + // SetBucketState + void onEncode(GBBuf&, const api::SetBucketStateCommand&) const override; + void onEncode(GBBuf&, const api::SetBucketStateReply&) const override; + SCmd::UP onDecodeSetBucketStateCommand(BBuf&) const override; + SRep::UP onDecodeSetBucketStateReply(const SCmd&, BBuf&) const override; + + // CreateVisitor + void onEncode(GBBuf&, const api::CreateVisitorCommand&) const override; + void onEncode(GBBuf&, const api::CreateVisitorReply&) const override; + SCmd::UP onDecodeCreateVisitorCommand(BBuf&) const override; + SRep::UP onDecodeCreateVisitorReply(const SCmd&, BBuf&) const override; + + // DestroyVisitor + void onEncode(GBBuf&, const api::DestroyVisitorCommand&) const override; + void onEncode(GBBuf&, const api::DestroyVisitorReply&) const override; + SCmd::UP onDecodeDestroyVisitorCommand(BBuf&) const override; + SRep::UP onDecodeDestroyVisitorReply(const SCmd&, BBuf&) const override; + + // RemoveLocation + void onEncode(GBBuf&, const api::RemoveLocationCommand&) const override; + void onEncode(GBBuf&, const api::RemoveLocationReply&) const override; + SCmd::UP onDecodeRemoveLocationCommand(BBuf&) const override; + SRep::UP onDecodeRemoveLocationReply(const SCmd&, BBuf&) const override; + +private: + template <typename ProtobufType, typename Func> + std::unique_ptr<api::StorageCommand> decode_request(document::ByteBuffer& in_buf, Func&& f) const; + template <typename ProtobufType, typename Func> + std::unique_ptr<api::StorageReply> decode_response(document::ByteBuffer& in_buf, Func&& f) const; + template <typename ProtobufType, typename Func> + std::unique_ptr<api::StorageCommand> decode_bucket_request(document::ByteBuffer& in_buf, Func&& f) const; + template <typename ProtobufType, typename Func> + std::unique_ptr<api::StorageReply> decode_bucket_response(document::ByteBuffer& in_buf, Func&& f) const; + template <typename ProtobufType, typename Func> + std::unique_ptr<api::StorageReply> decode_bucket_info_response(document::ByteBuffer& in_buf, Func&& f) const; +}; + +} +} diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp index 7e6be0a84f5..7bc6333762b 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp @@ -20,7 +20,8 @@ StorageProtocol::StorageProtocol(const std::shared_ptr<const document::DocumentT : _serializer5_0(repo, loadTypes), _serializer5_1(repo, loadTypes), _serializer5_2(repo, loadTypes), - _serializer6_0(repo, loadTypes) + _serializer6_0(repo, loadTypes), + _serializer7_0(repo, loadTypes) { } @@ -33,6 +34,7 @@ StorageProtocol::createPolicy(const mbus::string&, const mbus::string&) const } namespace { + vespalib::Version version7_0(7, 40, 5); vespalib::Version version6_0(6, 240, 0); vespalib::Version version5_2(5, 93, 30); vespalib::Version version5_1(5, 1, 0); @@ -106,8 +108,10 @@ StorageProtocol::encode(const vespalib::Version& version, } else { if (version < version6_0) { return encodeMessage(_serializer5_2, routable, message, version5_2, version); - } else { + } else if (version < version7_0) { return encodeMessage(_serializer6_0, routable, message, version6_0, version); + } else { + return encodeMessage(_serializer7_0, routable, message, version7_0, version); } } @@ -180,8 +184,10 @@ StorageProtocol::decode(const vespalib::Version & version, } else { if (version < version6_0) { return decodeMessage(_serializer5_2, data, type, version5_2, version); - } else { + } else if (version < version7_0) { return decodeMessage(_serializer6_0, data, type, version6_0, version); + } else { + return decodeMessage(_serializer7_0, data, type, version7_0, version); } } } catch (std::exception & e) { diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h index 1acd7c9675f..67ea121c340 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h +++ b/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h @@ -3,6 +3,7 @@ #include "protocolserialization5_2.h" #include "protocolserialization6_0.h" +#include "protocolserialization7.h" #include <vespa/messagebus/iprotocol.h> namespace storage::mbusprot { @@ -28,6 +29,7 @@ private: ProtocolSerialization5_1 _serializer5_1; ProtocolSerialization5_2 _serializer5_2; ProtocolSerialization6_0 _serializer6_0; + ProtocolSerialization7 _serializer7_0; }; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index d81c9f064b1..da3bd18440b 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -5,7 +5,7 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.OktaAccessToken; import com.yahoo.vespa.athenz.client.common.ClientBase; import com.yahoo.vespa.athenz.client.zms.bindings.AccessResponseEntity; @@ -55,7 +55,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void createTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + public void createTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token) { URI uri = zmsUrl.resolve(String.format("domain/%s/tenancy/%s", tenantDomain.getName(), providerService.getFullName())); HttpUriRequest request = RequestBuilder.put() .setUri(uri) @@ -66,7 +66,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void deleteTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token) { + public void deleteTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token) { URI uri = zmsUrl.resolve(String.format("domain/%s/tenancy/%s", tenantDomain.getName(), providerService.getFullName())); HttpUriRequest request = RequestBuilder.delete() .setUri(uri) @@ -76,7 +76,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { + public void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token) { URI uri = zmsUrl.resolve(String.format("domain/%s/provDomain/%s/provService/%s/resourceGroup/%s", tenantDomain.getName(), providerService.getDomainName(), providerService.getName(), resourceGroup)); HttpUriRequest request = RequestBuilder.put() .setUri(uri) @@ -87,7 +87,7 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override - public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, OktaAccessToken token) { + public void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, OktaAccessToken token) { URI uri = zmsUrl.resolve(String.format("domain/%s/provDomain/%s/provService/%s/resourceGroup/%s", tenantDomain.getName(), providerService.getDomainName(), providerService.getName(), resourceGroup)); HttpUriRequest request = RequestBuilder.delete() .setUri(uri) diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index cf044edeac0..e78478bc1a2 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -17,13 +17,13 @@ import java.util.Set; */ public interface ZmsClient extends AutoCloseable { - void createTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token); + void createTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token); - void deleteTenancy(AthenzDomain tenantDomain, AthenzService providerService, OktaAccessToken token); + void deleteTenancy(AthenzDomain tenantDomain, AthenzIdentity providerService, OktaAccessToken token); - void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token); + void createProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, Set<RoleAction> roleActions, OktaAccessToken token); - void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzService providerService, String resourceGroup, OktaAccessToken token); + void deleteProviderResourceGroup(AthenzDomain tenantDomain, AthenzIdentity providerService, String resourceGroup, OktaAccessToken token); boolean getMembership(AthenzRole role, AthenzIdentity identity); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java index dccd18fed61..a67bd4dcad6 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ProviderResourceGroupRolesRequestEntity.java @@ -33,7 +33,7 @@ public class ProviderResourceGroupRolesRequestEntity { @JsonProperty("resourceGroup") private final String resourceGroup; - public ProviderResourceGroupRolesRequestEntity(AthenzService providerService, AthenzDomain tenantDomain, Set<RoleAction> rolesActions, String resourceGroup) { + public ProviderResourceGroupRolesRequestEntity(AthenzIdentity providerService, AthenzDomain tenantDomain, Set<RoleAction> rolesActions, String resourceGroup) { this.domain = providerService.getDomainName(); this.service = providerService.getName(); this.tenant = tenantDomain.getName(); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java index 7883a505c71..6e1987130f2 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/TenancyRequestEntity.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.athenz.client.zms.bindings; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.yahoo.vespa.athenz.api.AthenzDomain; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import java.util.List; @@ -23,7 +24,7 @@ public class TenancyRequestEntity { @JsonInclude(JsonInclude.Include.NON_EMPTY) private final List<String> resourceGroups; - public TenancyRequestEntity(AthenzDomain tenantDomain, AthenzService providerService, List<String> resourceGroups) { + public TenancyRequestEntity(AthenzDomain tenantDomain, AthenzIdentity providerService, List<String> resourceGroups) { this.tenantDomain = tenantDomain.getName(); this.providerService = providerService.getFullName(); this.resourceGroups = resourceGroups; diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index 05395947fc1..ddba229d8d1 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -1,10 +1,10 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.client.zts; +import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzRole; -import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.AwsRole; import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials; import com.yahoo.vespa.athenz.api.NToken; @@ -22,7 +22,6 @@ import com.yahoo.vespa.athenz.client.zts.bindings.RoleTokenResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.TenantDomainsResponseEntity; import com.yahoo.vespa.athenz.client.zts.utils.IdentityCsrGenerator; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; -import com.yahoo.security.Pkcs10Csr; import org.apache.http.HttpResponse; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; @@ -65,8 +64,8 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override - public InstanceIdentity registerInstance(AthenzService providerIdentity, - AthenzService instanceIdentity, + public InstanceIdentity registerInstance(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String instanceId, String attestationData, boolean requestServiceToken, @@ -81,8 +80,8 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override - public InstanceIdentity refreshInstance(AthenzService providerIdentity, - AthenzService instanceIdentity, + public InstanceIdentity refreshInstance(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String instanceId, boolean requestServiceToken, Pkcs10Csr csr) { @@ -101,7 +100,7 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override - public Identity getServiceIdentity(AthenzService identity, String keyId, Pkcs10Csr csr) { + public Identity getServiceIdentity(AthenzIdentity identity, String keyId, Pkcs10Csr csr) { URI uri = ztsUrl.resolve(String.format("instance/%s/%s/refresh", identity.getDomainName(), identity.getName())); HttpUriRequest request = RequestBuilder.post() .setUri(uri) @@ -114,7 +113,7 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } @Override - public Identity getServiceIdentity(AthenzService identity, String keyId, KeyPair keyPair, String dnsSuffix) { + public Identity getServiceIdentity(AthenzIdentity identity, String keyId, KeyPair keyPair, String dnsSuffix) { Pkcs10Csr csr = new IdentityCsrGenerator(dnsSuffix).generateIdentityCsr(identity, keyPair); return getServiceIdentity(identity, keyId, csr); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java index 7b77fccfed6..efe244d500f 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java @@ -28,8 +28,8 @@ public interface ZtsClient extends AutoCloseable { * @param attestationData The signed identity documented serialized to a string. * @return A x509 certificate + service token (optional) */ - InstanceIdentity registerInstance(AthenzService providerIdentity, - AthenzService instanceIdentity, + InstanceIdentity registerInstance(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String instanceId, // TODO Remove this parameter (unused/unnecessary) String attestationData, boolean requestServiceToken, @@ -40,8 +40,8 @@ public interface ZtsClient extends AutoCloseable { * * @return A x509 certificate + service token (optional) */ - InstanceIdentity refreshInstance(AthenzService providerIdentity, - AthenzService instanceIdentity, + InstanceIdentity refreshInstance(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String instanceId, boolean requestServiceToken, Pkcs10Csr csr); @@ -51,7 +51,7 @@ public interface ZtsClient extends AutoCloseable { * * @return A x509 certificate with CA certificates */ - Identity getServiceIdentity(AthenzService identity, + Identity getServiceIdentity(AthenzIdentity identity, String keyId, Pkcs10Csr csr); @@ -60,7 +60,7 @@ public interface ZtsClient extends AutoCloseable { * * @return A x509 certificate with CA certificates */ - Identity getServiceIdentity(AthenzService identity, + Identity getServiceIdentity(AthenzIdentity identity, String keyId, KeyPair keyPair, String dnsSuffix); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRegisterInformation.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRegisterInformation.java index 49d9bb1ec5c..67a49059776 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRegisterInformation.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/bindings/InstanceRegisterInformation.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.athenz.client.zts.bindings; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.security.Pkcs10Csr; import com.yahoo.security.Pkcs10CsrUtils; @@ -32,8 +33,8 @@ public class InstanceRegisterInformation { @JsonProperty("token") private final boolean token; - public InstanceRegisterInformation(AthenzService providerIdentity, - AthenzService instanceIdentity, + public InstanceRegisterInformation(AthenzIdentity providerIdentity, + AthenzIdentity instanceIdentity, String attestationData, Pkcs10Csr csr, boolean requestServiceToken) { diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java index b2af2d732bf..d1383bd04fd 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/utils/IdentityCsrGenerator.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.client.zts.utils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.client.zts.ZtsClient; import com.yahoo.security.Pkcs10Csr; @@ -12,7 +13,7 @@ import java.security.KeyPair; import static com.yahoo.security.SignatureAlgorithm.SHA256_WITH_RSA; /** - * Generates a {@link Pkcs10Csr} instance for use with {@link ZtsClient#getServiceIdentity(AthenzService, String, Pkcs10Csr)} + * Generates a {@link Pkcs10Csr} instance for use with {@link ZtsClient#getServiceIdentity(AthenzIdentity, String, Pkcs10Csr)} * * @author bjorncs */ @@ -24,7 +25,7 @@ public class IdentityCsrGenerator { this.dnsSuffix = dnsSuffix; } - public Pkcs10Csr generateIdentityCsr(AthenzService identity, KeyPair keypair) { + public Pkcs10Csr generateIdentityCsr(AthenzIdentity identity, KeyPair keypair) { return Pkcs10CsrBuilder.fromKeypair(new X500Principal("CN=" + identity.getFullName()), keypair, SHA256_WITH_RSA) .addSubjectAlternativeName(String.format( "%s.%s.%s", diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java index 6b318fb16be..e5ed885b316 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/ServiceIdentityProvider.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.athenz.identity; import com.yahoo.container.jdisc.athenz.AthenzIdentityProvider; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import javax.net.ssl.SSLContext; @@ -13,6 +14,6 @@ import javax.net.ssl.SSLContext; * @author bjorncs */ public interface ServiceIdentityProvider { - AthenzService identity(); + AthenzIdentity identity(); SSLContext getIdentitySslContext(); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java index d8fa910aa73..2b0e50ed982 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identity/SiaIdentityProvider.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.athenz.identity; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.log.LogLevel; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.security.KeyStoreType; import com.yahoo.security.SslContextBuilder; @@ -33,7 +34,7 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde private static final Duration REFRESH_INTERVAL = Duration.ofHours(1); private final AtomicReference<SSLContext> sslContext = new AtomicReference<>(); - private final AthenzService service; + private final AthenzIdentity service; private final File privateKeyFile; private final File certificateFile; private final File trustStoreFile; @@ -48,7 +49,7 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde createScheduler()); } - public SiaIdentityProvider(AthenzService service, + public SiaIdentityProvider(AthenzIdentity service, Path siaPath, File trustStoreFile) { this(service, @@ -58,7 +59,7 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde createScheduler()); } - public SiaIdentityProvider(AthenzService service, + public SiaIdentityProvider(AthenzIdentity service, File privateKeyFile, File certificateFile, File trustStoreFile, @@ -81,7 +82,7 @@ public class SiaIdentityProvider extends AbstractComponent implements ServiceIde } @Override - public AthenzService identity() { + public AthenzIdentity identity() { return service; } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java index cd35a204b00..40f12b9c6db 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/utils/SiaUtils.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.utils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.security.KeyUtils; import com.yahoo.security.X509CertificateUtils; @@ -31,31 +32,31 @@ public class SiaUtils { private SiaUtils() {} - public static Path getPrivateKeyFile(AthenzService service) { + public static Path getPrivateKeyFile(AthenzIdentity service) { return getPrivateKeyFile(DEFAULT_SIA_DIRECTORY, service); } - public static Path getPrivateKeyFile(Path root, AthenzService service) { + public static Path getPrivateKeyFile(Path root, AthenzIdentity service) { return root .resolve("keys") .resolve(String.format("%s.%s.key.pem", service.getDomainName(), service.getName())); } - public static Path getCertificateFile(AthenzService service) { + public static Path getCertificateFile(AthenzIdentity service) { return getCertificateFile(DEFAULT_SIA_DIRECTORY, service); } - public static Path getCertificateFile(Path root, AthenzService service) { + public static Path getCertificateFile(Path root, AthenzIdentity service) { return root .resolve("certs") .resolve(String.format("%s.%s.cert.pem", service.getDomainName(), service.getName())); } - public static Optional<PrivateKey> readPrivateKeyFile(AthenzService service) { + public static Optional<PrivateKey> readPrivateKeyFile(AthenzIdentity service) { return readPrivateKeyFile(DEFAULT_SIA_DIRECTORY, service); } - public static Optional<PrivateKey> readPrivateKeyFile(Path root, AthenzService service) { + public static Optional<PrivateKey> readPrivateKeyFile(Path root, AthenzIdentity service) { try { Path privateKeyFile = getPrivateKeyFile(root, service); if (Files.notExists(privateKeyFile)) return Optional.empty(); @@ -65,11 +66,11 @@ public class SiaUtils { } } - public static Optional<X509Certificate> readCertificateFile(AthenzService service) { + public static Optional<X509Certificate> readCertificateFile(AthenzIdentity service) { return readCertificateFile(DEFAULT_SIA_DIRECTORY, service); } - public static Optional<X509Certificate> readCertificateFile(Path root, AthenzService service) { + public static Optional<X509Certificate> readCertificateFile(Path root, AthenzIdentity service) { try { Path certificateFile = getCertificateFile(root, service); if (Files.notExists(certificateFile)) return Optional.empty(); @@ -79,11 +80,11 @@ public class SiaUtils { } } - public static void writePrivateKeyFile(AthenzService service, PrivateKey privateKey) { + public static void writePrivateKeyFile(AthenzIdentity service, PrivateKey privateKey) { writePrivateKeyFile(DEFAULT_SIA_DIRECTORY, service, privateKey); } - public static void writePrivateKeyFile(Path root, AthenzService service, PrivateKey privateKey) { + public static void writePrivateKeyFile(Path root, AthenzIdentity service, PrivateKey privateKey) { try { Path privateKeyFile = getPrivateKeyFile(root, service); Files.createDirectories(privateKeyFile.getParent()); @@ -95,11 +96,11 @@ public class SiaUtils { } } - public static void writeCertificateFile(AthenzService service, X509Certificate certificate) { + public static void writeCertificateFile(AthenzIdentity service, X509Certificate certificate) { writeCertificateFile(DEFAULT_SIA_DIRECTORY, service, certificate); } - public static void writeCertificateFile(Path root, AthenzService service, X509Certificate certificate) { + public static void writeCertificateFile(Path root, AthenzIdentity service, X509Certificate certificate) { try { Path certificateFile = getCertificateFile(root, service); Files.createDirectories(certificateFile.getParent()); @@ -111,11 +112,11 @@ public class SiaUtils { } } - public static List<AthenzService> findSiaServices() { + public static List<AthenzIdentity> findSiaServices() { return findSiaServices(DEFAULT_SIA_DIRECTORY); } - public static List<AthenzService> findSiaServices(Path root) { + public static List<AthenzIdentity> findSiaServices(Path root) { String keyFileSuffix = ".key.pem"; Path keysDirectory = root.resolve("keys"); if ( ! Files.exists(keysDirectory)) diff --git a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java index f69e937f294..0e6aff1eeca 100644 --- a/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java +++ b/vespa-athenz/src/test/java/com/yahoo/vespa/athenz/utils/SiaUtilsTest.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.utils; +import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzService; import org.junit.Rule; import org.junit.Test; @@ -35,7 +36,7 @@ public class SiaUtilsTest { AthenzService barService = new AthenzService("my.domain.bar"); Files.createFile(SiaUtils.getPrivateKeyFile(siaRoot, barService)); - List<AthenzService> siaIdentities = SiaUtils.findSiaServices(siaRoot); + List<AthenzIdentity> siaIdentities = SiaUtils.findSiaServices(siaRoot); assertThat(siaIdentities.size(), equalTo(2)); assertThat(siaIdentities, hasItem(fooService)); assertThat(siaIdentities, hasItem(barService)); diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java index 3296cf13875..bd187ea3371 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/config/ConnectionParams.java @@ -36,7 +36,7 @@ public final class ConnectionParams { private long connectionTimeout = TimeUnit.SECONDS.toMillis(60); private final Multimap<String, String> headers = ArrayListMultimap.create(); private final Map<String, HeaderProvider> headerProviders = new HashMap<>(); - private int numPersistentConnectionsPerEndpoint = 8; + private int numPersistentConnectionsPerEndpoint = 1; private String proxyHost = null; private int proxyPort = 8080; private boolean useCompression = false; diff --git a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java index 98cd13a226d..b9219d8f267 100644 --- a/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java +++ b/vespa-http-client/src/main/java/com/yahoo/vespa/http/client/runner/CommandLineArguments.java @@ -169,7 +169,7 @@ public class CommandLineArguments { @Option(name = {"--numPersistentConnectionsPerEndpoint"}, description = "How many tcp connections to establish per endoint.)") - private int numPersistentConnectionsPerEndpoint = 16; + private int numPersistentConnectionsPerEndpoint = 4; @Option(name = {"--maxChunkSizeBytes"}, description = "How much data to send to gateway in each message.") @@ -226,7 +226,6 @@ public class CommandLineArguments { connectionParamsBuilder .setHostnameVerifier(insecure ? NoopHostnameVerifier.INSTANCE : SSLConnectionSocketFactory.getDefaultHostnameVerifier()) - .setNumPersistentConnectionsPerEndpoint(16) .setUseCompression(useCompressionArg) .setMaxRetries(noRetryArg ? 0 : 100) .setMinTimeBetweenRetries(retrydelayArg, TimeUnit.SECONDS) diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java index 39c1257816c..bca43902b9e 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/config/ConnectionParamsTest.java @@ -25,7 +25,7 @@ public class ConnectionParamsTest { ConnectionParams params = new ConnectionParams.Builder().build(); assertThat(params.getHeaders().isEmpty(), is(true)); - assertThat(params.getNumPersistentConnectionsPerEndpoint(), is(8)); + assertThat(params.getNumPersistentConnectionsPerEndpoint(), is(1)); assertThat(params.getSslContext(), nullValue()); } diff --git a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java index 53715259a0c..02509626176 100644 --- a/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java +++ b/vespa-http-client/src/test/java/com/yahoo/vespa/http/client/runner/CommandLineArgumentsTest.java @@ -18,7 +18,7 @@ import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.core.Is.is; import static org.hamcrest.core.IsNot.not; -import static org.junit.Assert.*; +import static org.junit.Assert.assertThat; public class CommandLineArgumentsTest { @@ -91,7 +91,7 @@ public class CommandLineArgumentsTest { assertThat(params.getClusters().get(0).getEndpoints().get(0).getPort(), is(4080)); assertThat(params.getClusters().get(0).getEndpoints().get(0).isUseSsl(), is(false)); assertThat(params.getConnectionParams().getUseCompression(), is(false)); - assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(16)); + assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(4)); assertThat(params.getFeedParams().getRoute(), is("default")); assertThat(params.getFeedParams().getDataFormat(), is(FeedParams.DataFormat.XML_UTF8)); assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(180000L)); @@ -106,6 +106,7 @@ public class CommandLineArgumentsTest { add("host", "hostValue"); add("port", "1234"); add("timeout", "2345"); + add("numPersistentConnectionsPerEndpoint", "7"); args.add("--useCompression"); args.add("--useDynamicThrottling"); add("maxpending", "3456"); @@ -125,6 +126,7 @@ public class CommandLineArgumentsTest { assertThat(params.getFeedParams().getLocalQueueTimeOut(), is(2345000L)); assertThat(params.getFeedParams().getMaxInFlightRequests(), is(3456)); assertThat(params.getFeedParams().getClientTimeout(TimeUnit.MILLISECONDS), is(2345000L)); + assertThat(params.getConnectionParams().getNumPersistentConnectionsPerEndpoint(), is(7)); } @Test diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java index 2b12c7cd78c..bfc4a611a5e 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/OperationHandlerImpl.java @@ -25,7 +25,6 @@ import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.metrics.simple.MetricReceiver; import com.yahoo.vdslib.VisitorOrdering; import com.yahoo.vespaclient.ClusterDef; -import com.yahoo.vespaclient.ClusterList; import com.yahoo.vespaxmlparser.VespaXMLFeedReader; import com.yahoo.yolean.concurrent.ConcurrentResourcePool; import com.yahoo.yolean.concurrent.ResourceFactory; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java index ecd4f7d1965..5072484567d 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/DenseBinaryFormat.java @@ -9,6 +9,8 @@ import com.yahoo.tensor.TensorType; import java.util.Iterator; import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Supplier; /** * Implementation of a dense binary format for a tensor on the form: @@ -22,40 +24,23 @@ import java.util.Optional; */ public class DenseBinaryFormat implements BinaryFormat { - static private final int DOUBLE_VALUE_TYPE = 0; // Not encoded as it is default, and you know the type when deserializing - static private final int FLOAT_VALUE_TYPE = 1; + private final TensorType.Value serializationValueType; - enum EncodeType {NO_DEFAULT, DOUBLE_IS_DEFAULT} - private final EncodeType encodeType; DenseBinaryFormat() { - encodeType = EncodeType.DOUBLE_IS_DEFAULT; + this(TensorType.Value.DOUBLE); } - DenseBinaryFormat(EncodeType encodeType) { - this.encodeType = encodeType; + DenseBinaryFormat(TensorType.Value serializationValueType) { + this.serializationValueType = serializationValueType; } @Override public void encode(GrowableByteBuffer buffer, Tensor tensor) { if ( ! ( tensor instanceof IndexedTensor)) throw new RuntimeException("The dense format is only supported for indexed tensors"); - encodeValueType(buffer, tensor.type().valueType()); encodeDimensions(buffer, (IndexedTensor)tensor); encodeCells(buffer, tensor); } - private void encodeValueType(GrowableByteBuffer buffer, TensorType.Value valueType) { - switch (valueType) { - case DOUBLE: - if (encodeType != EncodeType.DOUBLE_IS_DEFAULT) { - buffer.putInt1_4Bytes(DOUBLE_VALUE_TYPE); - } - break; - case FLOAT: - buffer.putInt1_4Bytes(FLOAT_VALUE_TYPE); - break; - } - } - private void encodeDimensions(GrowableByteBuffer buffer, IndexedTensor tensor) { buffer.putInt1_4Bytes(tensor.type().dimensions().size()); for (int i = 0; i < tensor.type().dimensions().size(); i++) { @@ -65,26 +50,17 @@ public class DenseBinaryFormat implements BinaryFormat { } private void encodeCells(GrowableByteBuffer buffer, Tensor tensor) { - switch (tensor.type().valueType()) { - case DOUBLE: - encodeCellsAsDouble(buffer, tensor); - break; - case FLOAT: - encodeCellsAsFloat(buffer, tensor); - break; + switch (serializationValueType) { + case DOUBLE: encodeCells(tensor, buffer::putDouble); break; + case FLOAT: encodeCells(tensor, (i) -> buffer.putFloat(i.floatValue())); break; } } - private void encodeCellsAsDouble(GrowableByteBuffer buffer, Tensor tensor) { + private void encodeCells(Tensor tensor, Consumer<Double> consumer) { Iterator<Double> i = tensor.valueIterator(); - while (i.hasNext()) - buffer.putDouble(i.next()); - } - - private void encodeCellsAsFloat(GrowableByteBuffer buffer, Tensor tensor) { - Iterator<Double> i = tensor.valueIterator(); - while (i.hasNext()) - buffer.putFloat(i.next().floatValue()); + while (i.hasNext()) { + consumer.accept(i.next()); + } } @Override @@ -93,40 +69,27 @@ public class DenseBinaryFormat implements BinaryFormat { DimensionSizes sizes; if (optionalType.isPresent()) { type = optionalType.get(); - TensorType serializedType = decodeType(buffer, type.valueType()); + if (type.valueType() != this.serializationValueType) { + throw new IllegalArgumentException("Tensor value type mismatch. Value type " + type.valueType() + + " is not " + this.serializationValueType); + } + TensorType serializedType = decodeType(buffer); if ( ! serializedType.isAssignableTo(type)) throw new IllegalArgumentException("Type/instance mismatch: A tensor of type " + serializedType + " cannot be assigned to type " + type); sizes = sizesFromType(serializedType); } else { - type = decodeType(buffer, TensorType.Value.DOUBLE); + type = decodeType(buffer); sizes = sizesFromType(type); } Tensor.Builder builder = Tensor.Builder.of(type, sizes); - decodeCells(type.valueType(), sizes, buffer, (IndexedTensor.BoundBuilder)builder); + decodeCells(sizes, buffer, (IndexedTensor.BoundBuilder)builder); return builder.build(); } - private TensorType decodeType(GrowableByteBuffer buffer, TensorType.Value valueType) { - TensorType.Value serializedValueType = TensorType.Value.DOUBLE; - if ((valueType != TensorType.Value.DOUBLE) || (encodeType != EncodeType.DOUBLE_IS_DEFAULT)) { - int type = buffer.getInt1_4Bytes(); - switch (type) { - case DOUBLE_VALUE_TYPE: - serializedValueType = TensorType.Value.DOUBLE; - break; - case FLOAT_VALUE_TYPE: - serializedValueType = TensorType.Value.FLOAT; - break; - default: - throw new IllegalArgumentException("Received tensor value type '" + serializedValueType + "'. Only 0(double), or 1(float) are legal."); - } - } - if (valueType != serializedValueType) { - throw new IllegalArgumentException("Expected " + valueType + ", got " + serializedValueType); - } - TensorType.Builder builder = new TensorType.Builder(serializedValueType); + private TensorType decodeType(GrowableByteBuffer buffer) { + TensorType.Builder builder = new TensorType.Builder(serializationValueType); int dimensionCount = buffer.getInt1_4Bytes(); for (int i = 0; i < dimensionCount; i++) builder.indexed(buffer.getUtf8String(), buffer.getInt1_4Bytes()); // XXX: Size truncation @@ -141,24 +104,16 @@ public class DenseBinaryFormat implements BinaryFormat { return builder.build(); } - private void decodeCells(TensorType.Value valueType, DimensionSizes sizes, GrowableByteBuffer buffer, IndexedTensor.BoundBuilder builder) { - switch (valueType) { - case DOUBLE: - decodeCellsAsDouble(sizes, buffer, builder); - break; - case FLOAT: - decodeCellsAsFloat(sizes, buffer, builder); - break; + private void decodeCells(DimensionSizes sizes, GrowableByteBuffer buffer, IndexedTensor.BoundBuilder builder) { + switch (serializationValueType) { + case DOUBLE: decodeCells(sizes, builder, buffer::getDouble); break; + case FLOAT: decodeCells(sizes, builder, () -> (double)buffer.getFloat()); break; } } - private void decodeCellsAsDouble(DimensionSizes sizes, GrowableByteBuffer buffer, IndexedTensor.BoundBuilder builder) { - for (long i = 0; i < sizes.totalSize(); i++) - builder.cellByDirectIndex(i, buffer.getDouble()); - } - private void decodeCellsAsFloat(DimensionSizes sizes, GrowableByteBuffer buffer, IndexedTensor.BoundBuilder builder) { + private void decodeCells(DimensionSizes sizes, IndexedTensor.BoundBuilder builder, Supplier<Double> supplier) { for (long i = 0; i < sizes.totalSize(); i++) - builder.cellByDirectIndex(i, buffer.getFloat()); + builder.cellByDirectIndex(i, supplier.get()); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java index 284dfea2141..bc247e5561f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/MixedBinaryFormat.java @@ -11,6 +11,8 @@ import com.yahoo.tensor.TensorType; import java.util.Iterator; import java.util.List; import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -21,6 +23,15 @@ import java.util.stream.Collectors; */ class MixedBinaryFormat implements BinaryFormat { + private final TensorType.Value serializationValueType; + + MixedBinaryFormat() { + this(TensorType.Value.DOUBLE); + } + MixedBinaryFormat(TensorType.Value serializationValueType) { + this.serializationValueType = serializationValueType; + } + @Override public void encode(GrowableByteBuffer buffer, Tensor tensor) { if ( ! ( tensor instanceof MixedTensor)) @@ -50,6 +61,13 @@ class MixedBinaryFormat implements BinaryFormat { } private void encodeCells(GrowableByteBuffer buffer, MixedTensor tensor) { + switch (serializationValueType) { + case DOUBLE: encodeCells(buffer, tensor, buffer::putDouble); break; + case FLOAT: encodeCells(buffer, tensor, (val) -> buffer.putFloat(val.floatValue())); break; + } + } + + private void encodeCells(GrowableByteBuffer buffer, MixedTensor tensor, Consumer<Double> consumer) { List<TensorType.Dimension> sparseDimensions = tensor.type().dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList()); long denseSubspaceSize = tensor.denseSubspaceSize(); if (sparseDimensions.size() > 0) { @@ -63,9 +81,9 @@ class MixedBinaryFormat implements BinaryFormat { new IllegalStateException("Dimension not found in address.")); buffer.putUtf8String(cell.getKey().label(index)); } - buffer.putDouble(cell.getValue()); + consumer.accept(cell.getValue()); for (int i = 1; i < denseSubspaceSize; ++i ) { - buffer.putDouble(cellIterator.next().getValue()); + consumer.accept(cellIterator.next().getValue()); } } } @@ -75,6 +93,10 @@ class MixedBinaryFormat implements BinaryFormat { TensorType type; if (optionalType.isPresent()) { type = optionalType.get(); + if (type.valueType() != this.serializationValueType) { + throw new IllegalArgumentException("Tensor value type mismatch. Value type " + type.valueType() + + " is not " + this.serializationValueType); + } TensorType serializedType = decodeType(buffer); if ( ! serializedType.isAssignableTo(type)) throw new IllegalArgumentException("Type/instance mismatch: A tensor of type " + serializedType + @@ -89,7 +111,7 @@ class MixedBinaryFormat implements BinaryFormat { } private TensorType decodeType(GrowableByteBuffer buffer) { - TensorType.Builder builder = new TensorType.Builder(); + TensorType.Builder builder = new TensorType.Builder(serializationValueType); int numMappedDimensions = buffer.getInt1_4Bytes(); for (int i = 0; i < numMappedDimensions; ++i) { builder.mapped(buffer.getUtf8String()); @@ -102,6 +124,13 @@ class MixedBinaryFormat implements BinaryFormat { } private void decodeCells(GrowableByteBuffer buffer, MixedTensor.BoundBuilder builder, TensorType type) { + switch (serializationValueType) { + case DOUBLE: decodeCells(buffer, builder, type, buffer::getDouble); break; + case FLOAT: decodeCells(buffer, builder, type, () -> (double)buffer.getFloat()); break; + } + } + + private void decodeCells(GrowableByteBuffer buffer, MixedTensor.BoundBuilder builder, TensorType type, Supplier<Double> supplier) { List<TensorType.Dimension> sparseDimensions = type.dimensions().stream().filter(d -> !d.isIndexed()).collect(Collectors.toList()); TensorType sparseType = MixedTensor.createPartialType(type.valueType(), sparseDimensions); long denseSubspaceSize = builder.denseSubspaceSize(); @@ -118,7 +147,7 @@ class MixedBinaryFormat implements BinaryFormat { sparseAddress.add(sparseDimension.name(), buffer.getUtf8String()); } for (long denseOffset = 0; denseOffset < denseSubspaceSize; denseOffset++) { - denseSubspace[(int)denseOffset] = buffer.getDouble(); + denseSubspace[(int)denseOffset] = supplier.get(); } builder.block(sparseAddress.build(), denseSubspace); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java index 190b31b7b35..cd671f824fa 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/SparseBinaryFormat.java @@ -8,8 +8,9 @@ import com.yahoo.tensor.TensorType; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Supplier; /** * Implementation of a sparse binary format for a tensor on the form: @@ -24,6 +25,15 @@ import java.util.Optional; */ class SparseBinaryFormat implements BinaryFormat { + private final TensorType.Value serializationValueType; + + SparseBinaryFormat() { + this(TensorType.Value.DOUBLE); + } + SparseBinaryFormat(TensorType.Value serializationValueType) { + this.serializationValueType = serializationValueType; + } + @Override public void encode(GrowableByteBuffer buffer, Tensor tensor) { encodeDimensions(buffer, tensor.type().dimensions()); @@ -39,10 +49,17 @@ class SparseBinaryFormat implements BinaryFormat { private void encodeCells(GrowableByteBuffer buffer, Tensor tensor) { buffer.putInt1_4Bytes((int)tensor.size()); // XXX: Size truncation + switch (serializationValueType) { + case DOUBLE: encodeCells(buffer, tensor, buffer::putDouble); break; + case FLOAT: encodeCells(buffer, tensor, (val) -> buffer.putFloat(val.floatValue())); break; + } + } + + private void encodeCells(GrowableByteBuffer buffer, Tensor tensor, Consumer<Double> consumer) { for (Iterator<Tensor.Cell> i = tensor.cellIterator(); i.hasNext(); ) { - Map.Entry<TensorAddress, Double> cell = i.next(); + Tensor.Cell cell = i.next(); encodeAddress(buffer, cell.getKey()); - buffer.putDouble(cell.getValue()); + consumer.accept(cell.getValue()); } } @@ -56,6 +73,10 @@ class SparseBinaryFormat implements BinaryFormat { TensorType type; if (optionalType.isPresent()) { type = optionalType.get(); + if (type.valueType() != this.serializationValueType) { + throw new IllegalArgumentException("Tensor value type mismatch. Value type " + type.valueType() + + " is not " + this.serializationValueType); + } TensorType serializedType = decodeType(buffer); if ( ! serializedType.isAssignableTo(type)) throw new IllegalArgumentException("Type/instance mismatch: A tensor of type " + serializedType + @@ -71,18 +92,25 @@ class SparseBinaryFormat implements BinaryFormat { private TensorType decodeType(GrowableByteBuffer buffer) { int numDimensions = buffer.getInt1_4Bytes(); - TensorType.Builder builder = new TensorType.Builder(); + TensorType.Builder builder = new TensorType.Builder(serializationValueType); for (int i = 0; i < numDimensions; ++i) builder.mapped(buffer.getUtf8String()); return builder.build(); } private void decodeCells(GrowableByteBuffer buffer, Tensor.Builder builder, TensorType type) { + switch (serializationValueType) { + case DOUBLE: decodeCells(buffer, builder, type, buffer::getDouble); break; + case FLOAT: decodeCells(buffer, builder, type, () -> (double)buffer.getFloat()); break; + } + } + + private void decodeCells(GrowableByteBuffer buffer, Tensor.Builder builder, TensorType type, Supplier<Double> supplier) { long numCells = buffer.getInt1_4Bytes(); // XXX: Size truncation for (long i = 0; i < numCells; ++i) { Tensor.Builder.CellBuilder cellBuilder = builder.cell(); decodeAddress(buffer, cellBuilder, type); - cellBuilder.value(buffer.getDouble()); + cellBuilder.value(supplier.get()); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java index 9b298f1dffb..bcff4392c9a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/TypedBinaryFormat.java @@ -27,32 +27,14 @@ public class TypedBinaryFormat { private static final int DENSE_BINARY_FORMAT_WITH_CELLTYPE = 6; private static final int MIXED_BINARY_FORMAT_WITH_CELLTYPE = 7; + private static final int DOUBLE_VALUE_TYPE = 0; // Not encoded as it is default, and you know the type when deserializing + private static final int FLOAT_VALUE_TYPE = 1; + public static byte[] encode(Tensor tensor) { GrowableByteBuffer buffer = new GrowableByteBuffer(); - if (tensor instanceof MixedTensor) { - buffer.putInt1_4Bytes(MIXED_BINARY_FORMAT_TYPE); - new MixedBinaryFormat().encode(buffer, tensor); - } - else if (tensor instanceof IndexedTensor) { - switch (tensor.type().valueType()) { - case DOUBLE: - buffer.putInt1_4Bytes(DENSE_BINARY_FORMAT_TYPE); - new DenseBinaryFormat(DenseBinaryFormat.EncodeType.DOUBLE_IS_DEFAULT).encode(buffer, tensor); - break; - default: - buffer.putInt1_4Bytes(DENSE_BINARY_FORMAT_WITH_CELLTYPE); - new DenseBinaryFormat(DenseBinaryFormat.EncodeType.NO_DEFAULT).encode(buffer, tensor); - break; - } - } - else { - buffer.putInt1_4Bytes(SPARSE_BINARY_FORMAT_TYPE); - new SparseBinaryFormat().encode(buffer, tensor); - } - buffer.flip(); - byte[] result = new byte[buffer.remaining()]; - buffer.get(result); - return result; + BinaryFormat encoder = getFormatEncoder(buffer, tensor); + encoder.encode(buffer, tensor); + return asByteArray(buffer); } /** @@ -64,14 +46,82 @@ public class TypedBinaryFormat { * @throws IllegalArgumentException if the tensor data was invalid */ public static Tensor decode(Optional<TensorType> type, GrowableByteBuffer buffer) { - int formatType = buffer.getInt1_4Bytes(); + BinaryFormat decoder = getFormatDecoder(buffer); + return decoder.decode(type, buffer); + } + + private static BinaryFormat getFormatEncoder(GrowableByteBuffer buffer, Tensor tensor) { + if (tensor instanceof MixedTensor && tensor.type().valueType() == TensorType.Value.DOUBLE) { + encodeFormatType(buffer, MIXED_BINARY_FORMAT_TYPE); + return new MixedBinaryFormat(); + } + if (tensor instanceof MixedTensor) { + encodeFormatType(buffer, MIXED_BINARY_FORMAT_WITH_CELLTYPE); + encodeValueType(buffer, tensor.type().valueType()); + return new MixedBinaryFormat(tensor.type().valueType()); + } + if (tensor instanceof IndexedTensor && tensor.type().valueType() == TensorType.Value.DOUBLE) { + encodeFormatType(buffer, DENSE_BINARY_FORMAT_TYPE); + return new DenseBinaryFormat(); + } + if (tensor instanceof IndexedTensor) { + encodeFormatType(buffer, DENSE_BINARY_FORMAT_WITH_CELLTYPE); + encodeValueType(buffer, tensor.type().valueType()); + return new DenseBinaryFormat(tensor.type().valueType()); + } + if (tensor.type().valueType() == TensorType.Value.DOUBLE) { + encodeFormatType(buffer, SPARSE_BINARY_FORMAT_TYPE); + return new SparseBinaryFormat(); + } + encodeFormatType(buffer, SPARSE_BINARY_FORMAT_WITH_CELLTYPE); + encodeValueType(buffer, tensor.type().valueType()); + return new SparseBinaryFormat(tensor.type().valueType()); + } + + private static BinaryFormat getFormatDecoder(GrowableByteBuffer buffer) { + int formatType = decodeFormatType(buffer); switch (formatType) { - case MIXED_BINARY_FORMAT_TYPE: return new MixedBinaryFormat().decode(type, buffer); - case SPARSE_BINARY_FORMAT_TYPE: return new SparseBinaryFormat().decode(type, buffer); - case DENSE_BINARY_FORMAT_TYPE: return new DenseBinaryFormat(DenseBinaryFormat.EncodeType.DOUBLE_IS_DEFAULT).decode(type, buffer); - case DENSE_BINARY_FORMAT_WITH_CELLTYPE: return new DenseBinaryFormat(DenseBinaryFormat.EncodeType.NO_DEFAULT).decode(type, buffer); - default: throw new IllegalArgumentException("Binary format type " + formatType + " is unknown"); + case SPARSE_BINARY_FORMAT_TYPE: return new SparseBinaryFormat(); + case DENSE_BINARY_FORMAT_TYPE: return new DenseBinaryFormat(); + case MIXED_BINARY_FORMAT_TYPE: return new MixedBinaryFormat(); + case SPARSE_BINARY_FORMAT_WITH_CELLTYPE: return new SparseBinaryFormat(decodeValueType(buffer)); + case DENSE_BINARY_FORMAT_WITH_CELLTYPE: return new DenseBinaryFormat(decodeValueType(buffer)); + case MIXED_BINARY_FORMAT_WITH_CELLTYPE: return new MixedBinaryFormat(decodeValueType(buffer)); } + throw new IllegalArgumentException("Binary format type " + formatType + " is unknown"); + } + + private static void encodeFormatType(GrowableByteBuffer buffer, int formatType) { + buffer.putInt1_4Bytes(formatType); + } + + private static int decodeFormatType(GrowableByteBuffer buffer) { + return buffer.getInt1_4Bytes(); + } + + private static void encodeValueType(GrowableByteBuffer buffer, TensorType.Value valueType) { + switch (valueType) { + case DOUBLE: buffer.putInt1_4Bytes(DOUBLE_VALUE_TYPE); break; + case FLOAT: buffer.putInt1_4Bytes(FLOAT_VALUE_TYPE); break; + default: + throw new IllegalArgumentException("Attempt to encode unknown tensor value type: " + valueType); + } + } + + private static TensorType.Value decodeValueType(GrowableByteBuffer buffer) { + int valueType = buffer.getInt1_4Bytes(); + switch (valueType) { + case DOUBLE_VALUE_TYPE: return TensorType.Value.DOUBLE; + case FLOAT_VALUE_TYPE: return TensorType.Value.FLOAT; + } + throw new IllegalArgumentException("Received tensor value type '" + valueType + "'. Only 0(double), or 1(float) are legal."); + } + + private static byte[] asByteArray(GrowableByteBuffer buffer) { + buffer.flip(); + byte[] result = new byte[buffer.remaining()]; + buffer.get(result); + return result; } } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/MixedBinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/MixedBinaryFormatTestCase.java index 33dfca017f4..69ef4922d8d 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/MixedBinaryFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/MixedBinaryFormatTestCase.java @@ -77,6 +77,12 @@ public class MixedBinaryFormatTestCase { assertSerialization(tensor); } + @Test + public void testSerializationOfDifferentValueTypes() { + assertSerialization("tensor<double>(x{},y[2]):{{x:0,y:0}:2.0, {x:0,y:1}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0}"); + assertSerialization("tensor<float>(x{},y[2]):{{x:0,y:0}:2.0, {x:0,y:1}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0}"); + } + private void assertSerialization(String tensorString) { assertSerialization(Tensor.from(tensorString)); } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/serialization/SparseBinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/serialization/SparseBinaryFormatTestCase.java index f895b64379b..9074579094c 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/serialization/SparseBinaryFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/serialization/SparseBinaryFormatTestCase.java @@ -1,7 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.tensor.serialization; -import com.google.common.collect.Sets; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -9,7 +8,6 @@ import org.junit.Test; import java.util.Arrays; import java.util.Optional; -import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -55,6 +53,25 @@ public class SparseBinaryFormatTestCase { Arrays.toString(TypedBinaryFormat.encode(Tensor.from("tensor(xy{},z{}):{{xy:ab,z:e}:2.0,{xy:cd,z:e}:3.0}")))); } + @Test + public void requireThatFloatSerializationFormatDoNotChange() { + byte[] encodedTensor = new byte[] {5, // binary format type + 1, // float type + 2, // num dimensions + 2, (byte)'x', (byte)'y', 1, (byte)'z', // dimensions + 2, // num cells, + 2, (byte)'a', (byte)'b', 1, (byte)'e', 64, 0, 0, 0, // cell 0 + 2, (byte)'c', (byte)'d', 1, (byte)'e', 64, 64, 0, 0}; // cell 1 + assertEquals(Arrays.toString(encodedTensor), + Arrays.toString(TypedBinaryFormat.encode(Tensor.from("tensor<float>(xy{},z{}):{{xy:ab,z:e}:2.0,{xy:cd,z:e}:3.0}")))); + } + + @Test + public void testSerializationOfDifferentValueTypes() { + assertSerialization("tensor<double>(x{},y{}):{{x:0,y:0}:2.0, {x:0,y:1}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0}"); + assertSerialization("tensor<float>(x{},y{}):{{x:0,y:0}:2.0, {x:0,y:1}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0}"); + } + private void assertSerialization(String tensorString) { assertSerialization(Tensor.from(tensorString)); } |