diff options
26 files changed, 540 insertions, 24 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 42af94b1d8e..a1db3c43f1c 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -187,6 +187,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addSimpleComponent("com.yahoo.container.jdisc.ContainerThreadFactory"); addSimpleComponent("com.yahoo.container.handler.VipStatus"); addSimpleComponent(com.yahoo.container.handler.ClustersStatus.class.getName()); + addSimpleComponent("com.yahoo.container.jdisc.DisabledConnectionLogProvider"); addJaxProviders(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 58fd5a77067..0c0c3f59e88 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -31,7 +31,6 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.container.logging.FileConnectionLog; -import com.yahoo.jdisc.http.server.jetty.VoidConnectionLog; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.searchdefinition.derived.RankProfileList; @@ -350,8 +349,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Add connection log if access log is configured if (cluster.getAllComponents().stream().anyMatch(component -> component instanceof AccessLogComponent)) { cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "qrs")); - } else { - cluster.addComponent(new ConnectionLogComponent(cluster, VoidConnectionLog.class, "qrs")); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index 56f09eefe82..f3f3b2b1076 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -8,16 +8,15 @@ import com.yahoo.container.logging.ConnectionLogConfig; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.container.logging.JSONAccessLog; import com.yahoo.container.logging.VespaAccessLog; -import com.yahoo.jdisc.http.server.jetty.VoidConnectionLog; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; import org.junit.Test; import org.w3c.dom.Element; import static com.yahoo.text.StringUtilities.quote; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertEquals; /** * @author gjoranv @@ -127,8 +126,6 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { nodesXml, "</container>" ); createModel(root, clusterElem); - Component<?, ?> voidConnectionLogComponent = getContainerComponent("default", VoidConnectionLog.class.getName()); - assertNotNull(voidConnectionLogComponent); Component<?, ?> fileConnectionLogComponent = getContainerComponent("default", FileConnectionLog.class.getName()); assertNull(fileConnectionLogComponent); } diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index 26650cb93a8..236602b7767 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -16,11 +16,9 @@ import com.yahoo.container.di.config.SubscriberFactory; import com.yahoo.container.di.osgi.BundleClasses; import com.yahoo.container.di.osgi.OsgiUtil; import com.yahoo.container.logging.AccessLog; -import com.yahoo.container.logging.ConnectionLog; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.jdisc.handler.RequestHandler; -import com.yahoo.jdisc.http.server.jetty.VoidConnectionLog; import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.jdisc.service.ServerProvider; import com.yahoo.osgi.OsgiImpl; @@ -50,6 +48,9 @@ public class HandlersConfigurerDi { private static final Logger log = Logger.getLogger(HandlersConfigurerDi.class.getName()); + private static final Executor fallbackExecutor = Executors.newCachedThreadPool( + ThreadFactoryFactory.getThreadFactory("HandlersConfigurerDI")); + private final com.yahoo.container.Container vespaContainer; private final Container container; @@ -140,11 +141,12 @@ public class HandlersConfigurerDi { return discInjector.createChildInjector(new AbstractModule() { @Override protected void configure() { + // Provide a singleton instance for all component fallbacks, + // otherwise fallback injection may lead to a cascade of components requiring reconstruction. bind(com.yahoo.container.Container.class).toInstance(vespaContainer); bind(com.yahoo.statistics.Statistics.class).toInstance(Statistics.nullImplementation); - bind(AccessLog.class).toInstance(new AccessLog(new ComponentRegistry<>())); - bind(Executor.class).toInstance(Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("HandlersConfigurerDI"))); - bind(ConnectionLog.class).toInstance(new VoidConnectionLog()); + bind(AccessLog.class).toInstance(AccessLog.NONE_INSTANCE); + bind(Executor.class).toInstance(fallbackExecutor); if (vespaContainer.getFileAcquirer() != null) bind(com.yahoo.filedistribution.fileacquirer.FileAcquirer.class).toInstance(vespaContainer.getFileAcquirer()); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/DisabledConnectionLogProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/DisabledConnectionLogProvider.java new file mode 100644 index 00000000000..53e479652ab --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/DisabledConnectionLogProvider.java @@ -0,0 +1,18 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.container.logging.ConnectionLog; +import com.yahoo.jdisc.http.server.jetty.VoidConnectionLog; + +/** + * @author bjorncs + */ +public class DisabledConnectionLogProvider implements Provider<ConnectionLog> { + + private static final ConnectionLog INSTANCE = new VoidConnectionLog(); + + @Override public ConnectionLog get() { return INSTANCE; } + @Override public void deconstruct() {} + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 76ac0611328..2a8dc34ea72 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -36,7 +36,8 @@ enum PathGroup { "/zone/v2/{*}", "/routing/v1/", "/routing/v1/status/environment/{*}", - "/routing/v1/inactive/environment/{*}"), + "/routing/v1/inactive/environment/{*}", + "/state/v1/{*}"), /** Paths used for creating and reading user resources. */ user(PathPrefix.api, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 7d3cb11718a..aca3da1ffe8 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -12,7 +12,12 @@ import java.util.List; import java.util.Optional; import java.util.TreeMap; -import static com.yahoo.vespa.flags.FetchVector.Dimension.*; +import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; +import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; +import static com.yahoo.vespa.flags.FetchVector.Dimension.TENANT_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.VESPA_VERSION; +import static com.yahoo.vespa.flags.FetchVector.Dimension.ZONE_ID; /** * Definitions of feature flags. @@ -232,7 +237,7 @@ public class Flags { "sync-host-logs-to-s3-bucket", "", List.of("andreer", "valerijf"), "2021-02-10", "2021-03-01", "Host-admin should sync host logs to an S3 bucket named by this flag. If left empty, sync is disabled", "Takes effect on next run of S3 log sync task in host-admin", - APPLICATION_ID); + APPLICATION_ID, NODE_TYPE); public static final UnboundIntFlag CLUSTER_CONTROLLER_MAX_HEAP_SIZE_IN_MB = defineIntFlag( "cluster-controller-max-heap-size-in-mb", 512, diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java index cdb4febb775..2d46c53bca7 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java @@ -13,6 +13,8 @@ import com.yahoo.component.provider.ComponentRegistry; */ public class AccessLog implements RequestLog { + public static final AccessLog NONE_INSTANCE = new AccessLog(new ComponentRegistry<>()); + private final ComponentRegistry<RequestLogHandler> implementers; @Inject @@ -21,7 +23,7 @@ public class AccessLog implements RequestLog { } public static AccessLog voidAccessLog() { - return new AccessLog(new ComponentRegistry<>()); + return NONE_INSTANCE; } @Override diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncClient.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncClient.java new file mode 100644 index 00000000000..64684a4b3e7 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncClient.java @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.util.List; + +/** + * @author freva + */ +public interface SyncClient { + + /** + * Syncs the given files, will only upload each file once. + * + * @param context context used to log which files were synced + * @param syncFileInfos list of files and their metadata to sync + * @param limit max number of files to upload for this invocation, to avoid blocking for too long + * @return true iff any files were uploaded + */ + boolean sync(TaskContext context, List<SyncFileInfo> syncFileInfos, int limit); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java new file mode 100644 index 00000000000..3b88d28613c --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java @@ -0,0 +1,83 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * @author freva + */ +public class SyncFileInfo { + + private final String bucketName; + private final Path srcPath; + private final Path destPath; + private final boolean compressWithZstd; + + private SyncFileInfo(String bucketName, Path srcPath, Path destPath, boolean compressWithZstd) { + this.bucketName = bucketName; + this.srcPath = srcPath; + this.destPath = destPath; + this.compressWithZstd = compressWithZstd; + } + + public String bucketName() { + return bucketName; + } + + public Path srcPath() { + return srcPath; + } + + public Path destPath() { + return destPath; + } + + public InputStream inputStream() throws IOException { + InputStream is = Files.newInputStream(srcPath); + if (compressWithZstd) return new ZstdCompressingInputStream(is, 4 << 20); + return is; + } + + + public static SyncFileInfo tenantVespaLog(String bucketName, ApplicationId applicationId, HostName hostName, Path vespaLogFile) { + return new SyncFileInfo(bucketName, vespaLogFile, destination(applicationId, hostName, "logs/vespa", vespaLogFile, ".zst"), true); + } + + public static SyncFileInfo tenantAccessLog(String bucketName, ApplicationId applicationId, HostName hostName, Path accessLogFile) { + return new SyncFileInfo(bucketName, accessLogFile, destination(applicationId, hostName, "logs/access", accessLogFile, null), false); + } + + public static SyncFileInfo infrastructureVespaLog(String bucketName, HostName hostName, Path vespaLogFile) { + return new SyncFileInfo(bucketName, vespaLogFile, destination(null, hostName, "logs/vespa", vespaLogFile, ".zst"), true); + } + + public static SyncFileInfo infrastructureAccessLog(String bucketName, HostName hostName, Path accessLogFile) { + return new SyncFileInfo(bucketName, accessLogFile, destination(null, hostName, "logs/access", accessLogFile, null), false); + } + + private static Path destination(ApplicationId app, HostName hostName, String dir, Path filename, String extension) { + StringBuilder sb = new StringBuilder(100).append('/'); + + if (app == null) sb.append("infrastructure"); + else sb.append(app.tenant().value()).append('.').append(app.application().value()).append('.').append(app.instance().value()); + + sb.append('/'); + for (char c: hostName.value().toCharArray()) { + if (c == '.') break; + sb.append(c); + } + + sb.append('/').append(dir).append('/').append(filename.getFileName().toString()); + + if (extension != null) sb.append(extension); + + return Paths.get(sb.toString()); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java new file mode 100644 index 00000000000..0cf8eaa9983 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java @@ -0,0 +1,75 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import com.yahoo.compress.ZstdCompressor; + +import java.io.IOException; +import java.io.InputStream; + +/** + * InputStream that outputs given InputStream compressed with the ZStandard. + * + * @author freva + */ +public class ZstdCompressingInputStream extends InputStream { + + public static final int DEFAULT_INPUT_BUFFER_SIZE = 8 * 1024; + static final ZstdCompressor compressor = new ZstdCompressor(); + + private final InputStream is; + private final byte[] inputBuffer; + private final byte[] outputBuffer; + + private int outputPosition = 0; + private int outputLength = 0; + private boolean isClosed = false; + + public ZstdCompressingInputStream(InputStream is, int inputBufferSize) { + this.is = is; + this.inputBuffer = new byte[inputBufferSize]; + this.outputBuffer = new byte[ZstdCompressor.getMaxCompressedLength(inputBufferSize)]; + } + + public ZstdCompressingInputStream(InputStream is) { + this(is, DEFAULT_INPUT_BUFFER_SIZE); + } + + @Override + public int read() throws IOException { + throwIfClosed(); + + if (outputPosition >= outputLength) { + int readLength = is.read(inputBuffer); + if (readLength == -1) + return -1; + + outputLength = compressor.compress(inputBuffer, 0, readLength, outputBuffer, 0, outputBuffer.length); + outputPosition = 0; + } + + return Byte.toUnsignedInt(outputBuffer[outputPosition++]); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int first = read(); + if (first == -1) return -1; + + b[off++] = (byte) first; + len = Math.min(Math.min(len, outputLength - outputPosition), b.length - off); + System.arraycopy(outputBuffer, outputPosition, b, off, len); + outputPosition += len; + return len + 1; + } + + @Override + public void close() throws IOException { + throwIfClosed(); + is.close(); + isClosed = true; + } + + private void throwIfClosed() { + if (isClosed) throw new IllegalArgumentException("Input stream is already closed"); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/package-info.java new file mode 100644 index 00000000000..fc197450492 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author freva + */ +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java new file mode 100644 index 00000000000..0d596a46d77 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java @@ -0,0 +1,79 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +/** + * @author freva + */ +public class SyncFileInfoTest { + + private static final FileSystem fileSystem = TestFileSystem.create(); + + private static final String bucket = "logs-region-acdf21"; + private static final ApplicationId application = ApplicationId.from("tenant", "application", "instance"); + private static final HostName hostname = HostName.from("h12352a.env.region-1.vespa.domain.example"); + private static final Path accessLogPath = fileSystem.getPath("/opt/vespa/logs/qrs/access.json-20210212.zst"); + private static final Path vespaLogPath = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12"); + + @Test + public void tenant_access_log() { + SyncFileInfo sfi = SyncFileInfo.tenantAccessLog(bucket, application, hostname, accessLogPath); + assertEquals(Paths.get("/tenant.application.instance/h12352a/logs/access/access.json-20210212.zst"), sfi.destPath()); + assertEquals(bucket, sfi.bucketName()); + assertNotEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi)); + } + + @Test + public void tenant_vespa_log() { + SyncFileInfo sfi = SyncFileInfo.tenantVespaLog(bucket, application, hostname, vespaLogPath); + assertEquals(Paths.get("/tenant.application.instance/h12352a/logs/vespa/vespa.log-2021-02-12.zst"), sfi.destPath()); + assertEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi)); + } + + @Test + public void infra_access_log() { + SyncFileInfo sfi = SyncFileInfo.infrastructureAccessLog(bucket, hostname, accessLogPath); + assertEquals(Paths.get("/infrastructure/h12352a/logs/access/access.json-20210212.zst"), sfi.destPath()); + assertNotEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi)); + } + + @Test + public void infra_vespa_log() { + SyncFileInfo sfi = SyncFileInfo.infrastructureVespaLog(bucket, hostname, vespaLogPath); + assertEquals(Paths.get("/infrastructure/h12352a/logs/vespa/vespa.log-2021-02-12.zst"), sfi.destPath()); + assertEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi)); + } + + @BeforeClass + public static void setup() throws IOException { + Files.createDirectories(vespaLogPath.getParent()); + Files.createFile(vespaLogPath); + Files.createDirectories(accessLogPath.getParent()); + Files.createFile(accessLogPath); + } + + private static Class<? extends InputStream> getInputStreamType(SyncFileInfo syncFileInfo) { + try (InputStream inputStream = syncFileInfo.inputStream()) { + return inputStream.getClass(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java new file mode 100644 index 00000000000..be0df437b7f --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java @@ -0,0 +1,45 @@ +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Random; + +import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.ZstdCompressingInputStream.compressor; +import static org.junit.Assert.assertArrayEquals; + +/** + * @author freva + */ +public class ZstdCompressingInputStreamTest { + + @Test + public void compression_test() throws Exception { + Random rnd = new Random(); + byte[] data = new byte[(int) (100_000 * (10 + rnd.nextDouble()))]; + rnd.nextBytes(data); + assertCompression(data, 1 << 14); + } + + private static void assertCompression(byte[] data, int bufferSize) { + ByteArrayInputStream bais = new ByteArrayInputStream(data); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(bais, bufferSize)) { + byte[] buffer = new byte[bufferSize]; + for (int nRead; (nRead = zcis.read(buffer, 0, buffer.length)) != -1; ) + baos.write(buffer, 0, nRead); + baos.flush(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + byte[] compressedData = baos.toByteArray(); + byte[] decompressedData = new byte[data.length]; + compressor.decompress(compressedData, 0, compressedData.length, decompressedData, 0, decompressedData.length); + + assertArrayEquals(data, decompressedData); + } +} diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index f1d910f2635..8efccf9b51b 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -26,6 +26,7 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/log/log.h> @@ -256,6 +257,7 @@ struct FixtureTraits { bool use_direct_tensor_attribute = false; bool enable_hnsw_index = false; bool use_mock_index = false; + bool use_mmap_file_allocator = false; FixtureTraits dense() && { use_dense_tensor_attribute = true; @@ -263,6 +265,11 @@ struct FixtureTraits { return *this; } + FixtureTraits mmap_file_allocator() && { + use_mmap_file_allocator = true; + return *this; + } + FixtureTraits hnsw() && { use_dense_tensor_attribute = true; enable_hnsw_index = true; @@ -327,6 +334,9 @@ struct Fixture { if (_cfg.tensorType().is_dense()) { _denseTensors = true; } + if (_traits.use_mmap_file_allocator) { + _cfg.setHuge(true); + } if (_traits.use_mock_index) { _index_factory = std::make_unique<MockNearestNeighborIndexFactory>(); } else { @@ -998,4 +1008,17 @@ TEST_F("NN blueprint handles strong filter triggering brute force search", Neare EXPECT_FALSE(bp->may_approximate()); } +TEST("Dense tensor attribute with huge flag uses mmap file allocator") +{ + vespalib::string basedir("mmap-file-allocator-factory-dir"); + vespalib::alloc::MmapFileAllocatorFactory::instance().setup(basedir); + { + Fixture f(vec_2d_spec, FixtureTraits().dense().mmap_file_allocator()); + vespalib::string allocator_dir(basedir + "/0.my_attr"); + EXPECT_TRUE(vespalib::isDirectory(allocator_dir)); + } + vespalib::alloc::MmapFileAllocatorFactory::instance().setup(""); + vespalib::rmdir(basedir, true); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp index b5d4fb3130f..75e231a815c 100644 --- a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp +++ b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp @@ -2,6 +2,7 @@ #include <vespa/log/log.h> LOG_SETUP("dense_tensor_store_test"); #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/memory_allocator.h> #include <vespa/searchlib/tensor/dense_tensor_store.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> @@ -27,7 +28,7 @@ struct Fixture { DenseTensorStore store; Fixture(const vespalib::string &tensorType) - : store(ValueType::from_spec(tensorType)) + : store(ValueType::from_spec(tensorType), {}) {} void assertSetAndGetTensor(const TensorSpec &tensorSpec) { Value::UP expTensor = makeTensor(tensorSpec); diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index be127a8be3e..0f39816257f 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -10,6 +10,8 @@ #include <vespa/searchlib/attribute/load_utils.h> #include <vespa/searchlib/attribute/readerbase.h> #include <vespa/vespalib/data/slime/inserter.h> +#include <vespa/vespalib/util/memory_allocator.h> +#include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.tensor.dense_tensor_attribute"); @@ -72,6 +74,15 @@ can_use_index_save_file(const search::attribute::Config &config, const search::a return true; } +std::unique_ptr<vespalib::alloc::MemoryAllocator> +make_memory_allocator(const vespalib::string& name, bool huge) +{ + if (huge) { + return vespalib::alloc::MmapFileAllocatorFactory::instance().make_memory_allocator(name); + } + return {}; +} + } void @@ -104,7 +115,7 @@ DenseTensorAttribute::memory_usage() const DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg, const NearestNeighborIndexFactory& index_factory) : TensorAttribute(baseFileName, cfg, _denseTensorStore), - _denseTensorStore(cfg.tensorType()), + _denseTensorStore(cfg.tensorType(), make_memory_allocator(getName(), cfg.huge())), _index() { if (cfg.hnsw_index_params().has_value()) { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index bbffc7ec4a5..36a803aa806 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -3,6 +3,7 @@ #include "dense_tensor_store.h" #include <vespa/eval/eval/value.h> #include <vespa/vespalib/datastore/datastore.hpp> +#include <vespa/vespalib/util/memory_allocator.h> using vespalib::datastore::Handle; using vespalib::eval::Value; @@ -46,8 +47,9 @@ DenseTensorStore::TensorSizeCalc::alignedSize() const return my_align(bufSize(), DENSE_TENSOR_ALIGNMENT); } -DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc) - : vespalib::datastore::BufferType<char>(tensorSizeCalc.alignedSize(), MIN_BUFFER_ARRAYS, RefType::offsetSize()) +DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator) + : vespalib::datastore::BufferType<char>(tensorSizeCalc.alignedSize(), MIN_BUFFER_ARRAYS, RefType::offsetSize()), + _allocator(std::move(allocator)) {} DenseTensorStore::BufferType::~BufferType() = default; @@ -59,11 +61,17 @@ DenseTensorStore::BufferType::cleanHold(void *buffer, size_t offset, memset(static_cast<char *>(buffer) + offset, 0, numElems); } -DenseTensorStore::DenseTensorStore(const ValueType &type) +const vespalib::alloc::MemoryAllocator* +DenseTensorStore::BufferType::get_memory_allocator() const +{ + return _allocator.get(); +} + +DenseTensorStore::DenseTensorStore(const ValueType &type, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator) : TensorStore(_concreteStore), _concreteStore(), _tensorSizeCalc(type), - _bufferType(_tensorSizeCalc), + _bufferType(_tensorSizeCalc, std::move(allocator)), _type(type), _emptySpace() { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h index 3f27b4c5218..aa5a7993eaf 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h @@ -34,10 +34,12 @@ public: class BufferType : public vespalib::datastore::BufferType<char> { using CleanContext = vespalib::datastore::BufferType<char>::CleanContext; + std::unique_ptr<vespalib::alloc::MemoryAllocator> _allocator; public: - BufferType(const TensorSizeCalc &tensorSizeCalc); + BufferType(const TensorSizeCalc &tensorSizeCalc, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator); ~BufferType() override; void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; + const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; private: DataStoreType _concreteStore; @@ -53,7 +55,7 @@ private: setDenseTensor(const TensorType &tensor); public: - DenseTensorStore(const ValueType &type); + DenseTensorStore(const ValueType &type, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator); ~DenseTensorStore() override; const ValueType &type() const { return _type; } diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 92bb4f7622b..5bdeecfcb7f 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -132,6 +132,7 @@ vespa_define_module( src/tests/util/generationhandler src/tests/util/generationhandler_stress src/tests/util/md5 + src/tests/util/mmap_file_allocator_factory src/tests/util/rcuvector src/tests/util/reusable_set src/tests/valgrind diff --git a/vespalib/src/tests/util/mmap_file_allocator_factory/.gitignore b/vespalib/src/tests/util/mmap_file_allocator_factory/.gitignore new file mode 100644 index 00000000000..a18e9aac589 --- /dev/null +++ b/vespalib/src/tests/util/mmap_file_allocator_factory/.gitignore @@ -0,0 +1 @@ +/mmap-file-allocator-factory-dir diff --git a/vespalib/src/tests/util/mmap_file_allocator_factory/CMakeLists.txt b/vespalib/src/tests/util/mmap_file_allocator_factory/CMakeLists.txt new file mode 100644 index 00000000000..c6122659a22 --- /dev/null +++ b/vespalib/src/tests/util/mmap_file_allocator_factory/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_mmap_file_allocator_factory_test_app TEST + SOURCES + mmap_file_allocator_factory_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME vespalib_mmap_file_allocator_factory_test_app COMMAND vespalib_mmap_file_allocator_factory_test_app) diff --git a/vespalib/src/tests/util/mmap_file_allocator_factory/mmap_file_allocator_factory_test.cpp b/vespalib/src/tests/util/mmap_file_allocator_factory/mmap_file_allocator_factory_test.cpp new file mode 100644 index 00000000000..f5c0317f623 --- /dev/null +++ b/vespalib/src/tests/util/mmap_file_allocator_factory/mmap_file_allocator_factory_test.cpp @@ -0,0 +1,49 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/mmap_file_allocator_factory.h> +#include <vespa/vespalib/util/mmap_file_allocator.h> +#include <vespa/vespalib/util/memory_allocator.h> +#include <vespa/vespalib/gtest/gtest.h> + +using vespalib::alloc::MemoryAllocator; +using vespalib::alloc::MmapFileAllocator; +using vespalib::alloc::MmapFileAllocatorFactory; + +namespace { + +vespalib::string basedir("mmap-file-allocator-factory-dir"); + +bool is_mmap_file_allocator(const MemoryAllocator *allocator) +{ + return dynamic_cast<const MmapFileAllocator *>(allocator) != nullptr; +} + +} + +TEST(MmapFileAllocatorFactoryTest, empty_dir_gives_no_allocator) +{ + MmapFileAllocatorFactory::instance().setup(""); + auto allocator = MmapFileAllocatorFactory::instance().make_memory_allocator("foo"); + EXPECT_FALSE(allocator); +} + +TEST(MmapFileAllocatorFactoryTest, nonempty_dir_gives_allocator) +{ + MmapFileAllocatorFactory::instance().setup(basedir); + auto allocator0 = MmapFileAllocatorFactory::instance().make_memory_allocator("foo"); + auto allocator1 = MmapFileAllocatorFactory::instance().make_memory_allocator("bar"); + EXPECT_TRUE(is_mmap_file_allocator(allocator0.get())); + EXPECT_TRUE(is_mmap_file_allocator(allocator1.get())); + vespalib::string allocator0_dir(basedir + "/0.foo"); + vespalib::string allocator1_dir(basedir + "/1.bar"); + EXPECT_TRUE(isDirectory(allocator0_dir)); + EXPECT_TRUE(isDirectory(allocator1_dir)); + allocator0.reset(); + EXPECT_FALSE(isDirectory(allocator0_dir)); + allocator1.reset(); + EXPECT_FALSE(isDirectory(allocator1_dir)); + MmapFileAllocatorFactory::instance().setup(""); + rmdir(basedir, true); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index fcac7cde412..d934a7b38ca 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -32,6 +32,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT lz4compressor.cpp md5.c mmap_file_allocator.cpp + mmap_file_allocator_factory.cpp printable.cpp priority_queue.cpp random.cpp diff --git a/vespalib/src/vespa/vespalib/util/mmap_file_allocator_factory.cpp b/vespalib/src/vespa/vespalib/util/mmap_file_allocator_factory.cpp new file mode 100644 index 00000000000..a575c08309d --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/mmap_file_allocator_factory.cpp @@ -0,0 +1,44 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "mmap_file_allocator_factory.h" +#include "mmap_file_allocator.h" +#include <vespa/vespalib/stllike/asciistream.h> + +namespace vespalib::alloc { + +MmapFileAllocatorFactory::MmapFileAllocatorFactory() + : _dir_name(), + _generation(0) +{ +} + +MmapFileAllocatorFactory::~MmapFileAllocatorFactory() +{ +} + +void +MmapFileAllocatorFactory::setup(const vespalib::string& dir_name) +{ + _dir_name = dir_name; + _generation = 0; +} + +std::unique_ptr<MemoryAllocator> +MmapFileAllocatorFactory::make_memory_allocator(const vespalib::string& name) +{ + if (_dir_name.empty()) { + return {}; + } + vespalib::asciistream os; + os << _dir_name << "/" << _generation.fetch_add(1) << "." << name; + return std::make_unique<MmapFileAllocator>(os.str()); +}; + +MmapFileAllocatorFactory& +MmapFileAllocatorFactory::instance() +{ + static MmapFileAllocatorFactory instance; + return instance; +} + +} diff --git a/vespalib/src/vespa/vespalib/util/mmap_file_allocator_factory.h b/vespalib/src/vespa/vespalib/util/mmap_file_allocator_factory.h new file mode 100644 index 00000000000..b2586787845 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/mmap_file_allocator_factory.h @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <memory> +#include <atomic> + +namespace vespalib::alloc { + +class MemoryAllocator; + +/* + * Class for creating an mmap file allocator on demand. + */ +class MmapFileAllocatorFactory { + vespalib::string _dir_name; + std::atomic<uint64_t> _generation; + + MmapFileAllocatorFactory(); + ~MmapFileAllocatorFactory(); + MmapFileAllocatorFactory(const MmapFileAllocatorFactory &) = delete; + MmapFileAllocatorFactory& operator=(const MmapFileAllocatorFactory &) = delete; +public: + void setup(const vespalib::string &dir_name); + std::unique_ptr<MemoryAllocator> make_memory_allocator(const vespalib::string& name); + + static MmapFileAllocatorFactory& instance(); +}; + +} |