From acd4af1d2c19119132c6803f15fff70cb65b7d4f Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Fri, 6 Apr 2018 00:21:15 +0200 Subject: Remove memfile implementation. --- CMakeLists.txt | 1 - memfilepersistence/.gitignore | 1 - memfilepersistence/CMakeLists.txt | 37 - memfilepersistence/OWNERS | 1 - memfilepersistence/README | 4 - memfilepersistence/src/.gitignore | 4 - memfilepersistence/src/Doxyfile | 994 ----------------- memfilepersistence/src/tests/.gitignore | 8 - memfilepersistence/src/tests/CMakeLists.txt | 17 - .../src/tests/conformance/.gitignore | 2 - .../src/tests/conformance/CMakeLists.txt | 6 - .../tests/conformance/memfileconformancetest.cpp | 35 - memfilepersistence/src/tests/device/.gitignore | 2 - memfilepersistence/src/tests/device/CMakeLists.txt | 11 - .../src/tests/device/devicemanagertest.cpp | 126 --- .../src/tests/device/devicemappertest.cpp | 49 - .../src/tests/device/devicestest.cpp | 68 -- .../src/tests/device/mountpointlisttest.cpp | 242 ----- .../src/tests/device/partitionmonitortest.cpp | 206 ---- memfilepersistence/src/tests/helper/CMakeLists.txt | 7 - memfilepersistence/src/tests/helper/testhelper.cpp | 123 --- memfilepersistence/src/tests/helper/testhelper.h | 53 - memfilepersistence/src/tests/init/.gitignore | 2 - memfilepersistence/src/tests/init/CMakeLists.txt | 7 - .../src/tests/init/filescannertest.cpp | 491 --------- memfilepersistence/src/tests/mapper/.gitignore | 0 memfilepersistence/src/tests/spi/.gitignore | 2 - memfilepersistence/src/tests/spi/CMakeLists.txt | 23 - .../src/tests/spi/basicoperationhandlertest.cpp | 743 ------------- memfilepersistence/src/tests/spi/buffer_test.cpp | 74 -- .../src/tests/spi/buffered_file_writer_test.cpp | 78 -- .../src/tests/spi/iteratorhandlertest.cpp | 929 ---------------- .../src/tests/spi/joinoperationhandlertest.cpp | 504 --------- memfilepersistence/src/tests/spi/logginglazyfile.h | 81 -- memfilepersistence/src/tests/spi/memcachetest.cpp | 397 ------- .../src/tests/spi/memfileautorepairtest.cpp | 409 ------- memfilepersistence/src/tests/spi/memfiletest.cpp | 987 ----------------- .../src/tests/spi/memfiletestutils.cpp | 455 -------- .../src/tests/spi/memfiletestutils.h | 296 ------ .../src/tests/spi/memfilev1serializertest.cpp | 1100 ------------------- .../src/tests/spi/memfilev1verifiertest.cpp | 496 --------- memfilepersistence/src/tests/spi/options_builder.h | 52 - .../src/tests/spi/providerconformancetest.cpp | 56 - .../spi/shared_data_location_tracker_test.cpp | 111 -- .../src/tests/spi/simplememfileiobuffertest.cpp | 656 ------------ .../src/tests/spi/simulatedfailurefile.cpp | 55 - .../src/tests/spi/simulatedfailurefile.h | 47 - .../src/tests/spi/splitoperationhandlertest.cpp | 212 ---- memfilepersistence/src/tests/testrunner.cpp | 13 - memfilepersistence/src/tests/tools/.gitignore | 2 - memfilepersistence/src/tests/tools/CMakeLists.txt | 9 - .../src/tests/tools/dumpslotfiletest.cpp | 138 --- .../src/tests/tools/vdsdisktooltest.cpp | 104 -- .../src/vespa/memfilepersistence/.gitignore | 3 - .../src/vespa/memfilepersistence/CMakeLists.txt | 13 - .../src/vespa/memfilepersistence/common/.gitignore | 2 - .../vespa/memfilepersistence/common/CMakeLists.txt | 12 - .../memfilepersistence/common/config_aliases.h | 18 - .../common/config_lock_guard.cpp | 113 -- .../memfilepersistence/common/config_lock_guard.h | 98 -- .../vespa/memfilepersistence/common/configkeeper.h | 57 - .../memfilepersistence/common/environment.cpp | 120 --- .../vespa/memfilepersistence/common/environment.h | 132 --- .../vespa/memfilepersistence/common/exceptions.cpp | 59 -- .../vespa/memfilepersistence/common/exceptions.h | 132 --- .../common/filespecification.cpp | 35 - .../memfilepersistence/common/filespecification.h | 49 - .../vespa/memfilepersistence/common/options.cpp | 230 ---- .../src/vespa/memfilepersistence/common/options.h | 88 -- .../memfilepersistence/common/slotmatcher.cpp | 40 - .../vespa/memfilepersistence/common/slotmatcher.h | 89 -- .../src/vespa/memfilepersistence/common/types.cpp | 50 - .../src/vespa/memfilepersistence/common/types.h | 183 ---- .../src/vespa/memfilepersistence/device/.gitignore | 2 - .../vespa/memfilepersistence/device/CMakeLists.txt | 14 - .../src/vespa/memfilepersistence/device/device.cpp | 56 - .../src/vespa/memfilepersistence/device/device.h | 72 -- .../memfilepersistence/device/devicemanager.cpp | 210 ---- .../memfilepersistence/device/devicemanager.h | 68 -- .../memfilepersistence/device/devicemapper.cpp | 96 -- .../vespa/memfilepersistence/device/devicemapper.h | 102 -- .../vespa/memfilepersistence/device/directory.cpp | 145 --- .../vespa/memfilepersistence/device/directory.h | 68 -- .../src/vespa/memfilepersistence/device/disk.cpp | 41 - .../src/vespa/memfilepersistence/device/disk.h | 45 - .../vespa/memfilepersistence/device/ioevent.cpp | 130 --- .../src/vespa/memfilepersistence/device/ioevent.h | 83 -- .../memfilepersistence/device/mountpointlist.cpp | 643 ----------- .../memfilepersistence/device/mountpointlist.h | 133 --- .../vespa/memfilepersistence/device/partition.cpp | 61 -- .../vespa/memfilepersistence/device/partition.h | 56 - .../memfilepersistence/device/partitionmonitor.cpp | 389 ------- .../memfilepersistence/device/partitionmonitor.h | 151 --- .../src/vespa/memfilepersistence/init/.gitignore | 2 - .../vespa/memfilepersistence/init/CMakeLists.txt | 6 - .../vespa/memfilepersistence/init/filescanner.cpp | 240 ----- .../vespa/memfilepersistence/init/filescanner.h | 105 -- .../src/vespa/memfilepersistence/mapper/.gitignore | 2 - .../vespa/memfilepersistence/mapper/CMakeLists.txt | 16 - .../mapper/bucketdirectorymapper.cpp | 31 - .../mapper/bucketdirectorymapper.h | 40 - .../src/vespa/memfilepersistence/mapper/buffer.cpp | 32 - .../src/vespa/memfilepersistence/mapper/buffer.h | 55 - .../mapper/bufferedfilewriter.cpp | 219 ---- .../memfilepersistence/mapper/bufferedfilewriter.h | 116 -- .../vespa/memfilepersistence/mapper/fileinfo.cpp | 102 -- .../src/vespa/memfilepersistence/mapper/fileinfo.h | 134 --- .../mapper/locationreadplanner.cpp | 100 -- .../mapper/locationreadplanner.h | 59 -- .../mapper/mapperslotoperation.h | 61 -- .../mapper/memfile_v1_serializer.cpp | 1026 ------------------ .../mapper/memfile_v1_serializer.h | 55 - .../mapper/memfile_v1_verifier.cpp | 694 ------------ .../mapper/memfile_v1_verifier.h | 84 -- .../memfilepersistence/mapper/memfilemapper.cpp | 339 ------ .../memfilepersistence/mapper/memfilemapper.h | 109 -- .../mapper/serializationmetrics.cpp | 90 -- .../mapper/serializationmetrics.h | 47 - .../mapper/simplememfileiobuffer.cpp | 546 ---------- .../mapper/simplememfileiobuffer.h | 330 ------ .../memfilepersistence/mapper/slotreadplanner.cpp | 101 -- .../memfilepersistence/mapper/slotreadplanner.h | 59 -- .../mapper/uniqueslotgenerator.cpp | 84 -- .../mapper/uniqueslotgenerator.h | 67 -- .../memfilepersistence/mapper/versionserializer.h | 96 -- .../vespa/memfilepersistence/memfile/.gitignore | 2 - .../memfilepersistence/memfile/CMakeLists.txt | 11 - .../vespa/memfilepersistence/memfile/memfile.cpp | 1117 -------------------- .../src/vespa/memfilepersistence/memfile/memfile.h | 317 ------ .../memfilepersistence/memfile/memfilecache.cpp | 515 --------- .../memfilepersistence/memfile/memfilecache.h | 297 ------ .../memfile/memfilecompactor.cpp | 204 ---- .../memfilepersistence/memfile/memfilecompactor.h | 67 -- .../memfile/memfileiointerface.h | 80 -- .../vespa/memfilepersistence/memfile/memfileptr.h | 90 -- .../vespa/memfilepersistence/memfile/memslot.cpp | 142 --- .../src/vespa/memfilepersistence/memfile/memslot.h | 184 ---- .../memfile/shared_data_location_tracker.cpp | 21 - .../memfile/shared_data_location_tracker.h | 66 -- .../memfilepersistence/memfile/slotiterator.cpp | 106 -- .../memfilepersistence/memfile/slotiterator.h | 129 --- .../src/vespa/memfilepersistence/spi/.gitignore | 2 - .../vespa/memfilepersistence/spi/CMakeLists.txt | 15 - .../memfilepersistence/spi/cacheevictionguard.cpp | 29 - .../memfilepersistence/spi/cacheevictionguard.h | 46 - .../memfilepersistence/spi/iteratorhandler.cpp | 437 -------- .../vespa/memfilepersistence/spi/iteratorhandler.h | 240 ----- .../spi/joinoperationhandler.cpp | 160 --- .../memfilepersistence/spi/joinoperationhandler.h | 37 - .../memfilepersistence/spi/memfilepersistence.cpp | 11 - .../memfilepersistence/spi/memfilepersistence.h | 20 - .../spi/memfilepersistenceprovider.cpp | 914 ---------------- .../spi/memfilepersistenceprovider.h | 133 --- .../spi/memfilepersistenceprovidermetrics.cpp | 66 -- .../spi/memfilepersistenceprovidermetrics.h | 61 -- .../memfilepersistence/spi/operationhandler.cpp | 286 ----- .../memfilepersistence/spi/operationhandler.h | 121 --- .../spi/splitoperationhandler.cpp | 110 -- .../memfilepersistence/spi/splitoperationhandler.h | 41 - .../vespa/memfilepersistence/spi/threadlocals.cpp | 24 - .../vespa/memfilepersistence/spi/threadlocals.h | 55 - .../memfilepersistence/spi/threadmetricprovider.h | 18 - .../memfilepersistence/spi/visitorslotmatcher.cpp | 70 -- .../memfilepersistence/spi/visitorslotmatcher.h | 31 - .../src/vespa/memfilepersistence/tools/.gitignore | 4 - .../vespa/memfilepersistence/tools/CMakeLists.txt | 23 - .../memfilepersistence/tools/dumpslotfile.cpp | 358 ------- .../vespa/memfilepersistence/tools/dumpslotfile.h | 22 - .../memfilepersistence/tools/dumpslotfileapp.cpp | 28 - .../vespa/memfilepersistence/tools/vdsdiskapp.cpp | 29 - .../vespa/memfilepersistence/tools/vdsdisktool.cpp | 518 --------- .../vespa/memfilepersistence/tools/vdsdisktool.h | 18 - .../vespa/memfilepersistence/tools/vdsdisktool.pl | 47 - storageserver/CMakeLists.txt | 1 - storageserver/src/apps/storaged/storage.cpp | 5 +- storageserver/src/tests/storageservertest.cpp | 6 +- .../src/vespa/storageserver/app/CMakeLists.txt | 1 - .../app/memfileservicelayerprocess.cpp | 107 -- .../storageserver/app/memfileservicelayerprocess.h | 59 -- 179 files changed, 4 insertions(+), 27158 deletions(-) delete mode 100644 memfilepersistence/.gitignore delete mode 100644 memfilepersistence/CMakeLists.txt delete mode 100644 memfilepersistence/OWNERS delete mode 100644 memfilepersistence/README delete mode 100644 memfilepersistence/src/.gitignore delete mode 100644 memfilepersistence/src/Doxyfile delete mode 100644 memfilepersistence/src/tests/.gitignore delete mode 100644 memfilepersistence/src/tests/CMakeLists.txt delete mode 100644 memfilepersistence/src/tests/conformance/.gitignore delete mode 100644 memfilepersistence/src/tests/conformance/CMakeLists.txt delete mode 100644 memfilepersistence/src/tests/conformance/memfileconformancetest.cpp delete mode 100644 memfilepersistence/src/tests/device/.gitignore delete mode 100644 memfilepersistence/src/tests/device/CMakeLists.txt delete mode 100644 memfilepersistence/src/tests/device/devicemanagertest.cpp delete mode 100644 memfilepersistence/src/tests/device/devicemappertest.cpp delete mode 100644 memfilepersistence/src/tests/device/devicestest.cpp delete mode 100644 memfilepersistence/src/tests/device/mountpointlisttest.cpp delete mode 100644 memfilepersistence/src/tests/device/partitionmonitortest.cpp delete mode 100644 memfilepersistence/src/tests/helper/CMakeLists.txt delete mode 100644 memfilepersistence/src/tests/helper/testhelper.cpp delete mode 100644 memfilepersistence/src/tests/helper/testhelper.h delete mode 100644 memfilepersistence/src/tests/init/.gitignore delete mode 100644 memfilepersistence/src/tests/init/CMakeLists.txt delete mode 100644 memfilepersistence/src/tests/init/filescannertest.cpp delete mode 100644 memfilepersistence/src/tests/mapper/.gitignore delete mode 100644 memfilepersistence/src/tests/spi/.gitignore delete mode 100644 memfilepersistence/src/tests/spi/CMakeLists.txt delete mode 100644 memfilepersistence/src/tests/spi/basicoperationhandlertest.cpp delete mode 100644 memfilepersistence/src/tests/spi/buffer_test.cpp delete mode 100644 memfilepersistence/src/tests/spi/buffered_file_writer_test.cpp delete mode 100644 memfilepersistence/src/tests/spi/iteratorhandlertest.cpp delete mode 100644 memfilepersistence/src/tests/spi/joinoperationhandlertest.cpp delete mode 100644 memfilepersistence/src/tests/spi/logginglazyfile.h delete mode 100644 memfilepersistence/src/tests/spi/memcachetest.cpp delete mode 100644 memfilepersistence/src/tests/spi/memfileautorepairtest.cpp delete mode 100644 memfilepersistence/src/tests/spi/memfiletest.cpp delete mode 100644 memfilepersistence/src/tests/spi/memfiletestutils.cpp delete mode 100644 memfilepersistence/src/tests/spi/memfiletestutils.h delete mode 100644 memfilepersistence/src/tests/spi/memfilev1serializertest.cpp delete mode 100644 memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp delete mode 100644 memfilepersistence/src/tests/spi/options_builder.h delete mode 100644 memfilepersistence/src/tests/spi/providerconformancetest.cpp delete mode 100644 memfilepersistence/src/tests/spi/shared_data_location_tracker_test.cpp delete mode 100644 memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp delete mode 100644 memfilepersistence/src/tests/spi/simulatedfailurefile.cpp delete mode 100644 memfilepersistence/src/tests/spi/simulatedfailurefile.h delete mode 100644 memfilepersistence/src/tests/spi/splitoperationhandlertest.cpp delete mode 100644 memfilepersistence/src/tests/testrunner.cpp delete mode 100644 memfilepersistence/src/tests/tools/.gitignore delete mode 100644 memfilepersistence/src/tests/tools/CMakeLists.txt delete mode 100644 memfilepersistence/src/tests/tools/dumpslotfiletest.cpp delete mode 100644 memfilepersistence/src/tests/tools/vdsdisktooltest.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/.gitignore delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/CMakeLists.txt delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/.gitignore delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/CMakeLists.txt delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/config_aliases.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/config_lock_guard.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/config_lock_guard.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/configkeeper.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/environment.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/environment.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/exceptions.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/exceptions.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/filespecification.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/filespecification.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/options.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/options.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/slotmatcher.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/slotmatcher.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/types.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/common/types.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/.gitignore delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/CMakeLists.txt delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/device.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/device.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/directory.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/directory.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/disk.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/disk.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/ioevent.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/ioevent.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/partition.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/partition.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/device/partitionmonitor.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/init/.gitignore delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/init/CMakeLists.txt delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/init/filescanner.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/init/filescanner.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/.gitignore delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/CMakeLists.txt delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/bucketdirectorymapper.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/bucketdirectorymapper.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/buffer.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/buffer.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/bufferedfilewriter.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/fileinfo.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/fileinfo.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/locationreadplanner.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/locationreadplanner.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/mapperslotoperation.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_serializer.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/memfile_v1_verifier.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/memfilemapper.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/serializationmetrics.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/slotreadplanner.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/slotreadplanner.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/uniqueslotgenerator.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/uniqueslotgenerator.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/mapper/versionserializer.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/.gitignore delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/CMakeLists.txt delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/memfile.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecache.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecompactor.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/memfilecompactor.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/memfileiointerface.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/memfileptr.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/memslot.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/memslot.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/shared_data_location_tracker.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/slotiterator.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/memfile/slotiterator.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/.gitignore delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/CMakeLists.txt delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/cacheevictionguard.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/cacheevictionguard.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/iteratorhandler.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/joinoperationhandler.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistence.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistence.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovider.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/memfilepersistenceprovidermetrics.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/operationhandler.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/splitoperationhandler.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/splitoperationhandler.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/threadlocals.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/threadlocals.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/threadmetricprovider.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/visitorslotmatcher.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/spi/visitorslotmatcher.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/tools/.gitignore delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/tools/CMakeLists.txt delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfile.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/tools/dumpslotfileapp.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/tools/vdsdiskapp.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/tools/vdsdisktool.cpp delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/tools/vdsdisktool.h delete mode 100644 memfilepersistence/src/vespa/memfilepersistence/tools/vdsdisktool.pl delete mode 100644 storageserver/src/vespa/storageserver/app/memfileservicelayerprocess.cpp delete mode 100644 storageserver/src/vespa/storageserver/app/memfileservicelayerprocess.h diff --git a/CMakeLists.txt b/CMakeLists.txt index b30a54fb576..4b34def92c9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,7 +73,6 @@ add_subdirectory(logd) add_subdirectory(logserver) add_subdirectory(logforwarder) add_subdirectory(lowercasing_test) -add_subdirectory(memfilepersistence) add_subdirectory(messagebus) add_subdirectory(messagebus_test) add_subdirectory(metrics) diff --git a/memfilepersistence/.gitignore b/memfilepersistence/.gitignore deleted file mode 100644 index f3c7a7c5da6..00000000000 --- a/memfilepersistence/.gitignore +++ /dev/null @@ -1 +0,0 @@ -Makefile diff --git a/memfilepersistence/CMakeLists.txt b/memfilepersistence/CMakeLists.txt deleted file mode 100644 index a1fcd0028ac..00000000000 --- a/memfilepersistence/CMakeLists.txt +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - DEPENDS - vespadefaults - fastos - vespalog - vespalib - document - config_cloudconfig - configdefinitions - vdslib - persistence - storageframework - - LIBS - src/vespa/memfilepersistence - src/vespa/memfilepersistence/common - src/vespa/memfilepersistence/device - src/vespa/memfilepersistence/init - src/vespa/memfilepersistence/mapper - src/vespa/memfilepersistence/memfile - src/vespa/memfilepersistence/spi - src/vespa/memfilepersistence/tools - - TEST_DEPENDS - persistence_persistence_conformancetest - vdstestlib - - TESTS - src/tests - src/tests/conformance - src/tests/device - src/tests/init - src/tests/helper - src/tests/spi - src/tests/tools -) diff --git a/memfilepersistence/OWNERS b/memfilepersistence/OWNERS deleted file mode 100644 index dbcff24b338..00000000000 --- a/memfilepersistence/OWNERS +++ /dev/null @@ -1 +0,0 @@ -vekterli diff --git a/memfilepersistence/README b/memfilepersistence/README deleted file mode 100644 index 38a0e92febb..00000000000 --- a/memfilepersistence/README +++ /dev/null @@ -1,4 +0,0 @@ -This module contains Storage's persistence SPI implementation. - -It uses memory representation of files, which currently are persisted in -slotfiles on disk. diff --git a/memfilepersistence/src/.gitignore b/memfilepersistence/src/.gitignore deleted file mode 100644 index d6d89678e22..00000000000 --- a/memfilepersistence/src/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/Makefile.ini -/config_command.sh -/project.dsw -/memfilepersistence.mak diff --git a/memfilepersistence/src/Doxyfile b/memfilepersistence/src/Doxyfile deleted file mode 100644 index 671dc341e73..00000000000 --- a/memfilepersistence/src/Doxyfile +++ /dev/null @@ -1,994 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.2.18 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# General configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = Storage - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = ../doc - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, -# Finnish, French, German, Greek, Hungarian, Italian, Japanese, Japanese-en -# (Japanese with english messages), Korean, Norwegian, Polish, Portuguese, -# Romanian, Russian, Serbian, Slovak, Slovene, Spanish, Swedish and Ukrainian. - -OUTPUT_LANGUAGE = English - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = NO - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these class will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all inherited -# members of a class in the documentation of that class as if those members were -# ordinary class members. Constructors, destructors and assignment operators of -# the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = NO - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. It is allowed to use relative paths in the argument list. - -STRIP_FROM_PATH = - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower case letters. If set to YES upper case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# users are adviced to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explict @brief command for a brief description. - -JAVADOC_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -DETAILS_AT_TOP = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# reimplements. - -INHERIT_DOCS = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 4 - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consist of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. -# For instance some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = YES - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java sources -# only. Doxygen will then generate output that is more tailored for Java. -# For instance namespaces will be presented as packages, qualified scopes -# will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = storage - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx *.hpp -# *.h++ *.idl *.odl - -FILE_PATTERNS = *.h *.cpp - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or directories -# that are symbolic links (a Unix filesystem feature) are excluded from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. - -EXCLUDE_PATTERNS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command , where -# is the value of the INPUT_FILTER tag, and is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. - -INPUT_FILTER = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = NO - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet - -HTML_STYLESHEET = ../cpp/vespa_link.css - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output dir. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non empty doxygen will try to run -# the html help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the Html help documentation and to the tree view. - -TOC_EXPAND = NO - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript and frames is required (for instance Mozilla, Netscape 4.0+, -# or Internet explorer 4.0+). Note that for large projects the tree generation -# can take a very long time. In such cases it is better to disable this feature. -# Windows users are probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = YES - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be invoked. If left blank `latex' will be used as the default command name. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = NO - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = YES - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimised for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assigments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_XML = NO - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_PREDEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. - -PREDEFINED = - -# If the MACRO_EXPANSION and EXPAND_PREDEF_ONLY tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line, have an all uppercase name, and do not end with a semicolon. Such -# function macros are typically used for boiler-plate code, and will confuse the -# parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES tag can be used to specify one or more tagfiles. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in Html, RTF and LaTeX) for classes with base or -# super classes. Setting the tag to NO turns the diagrams off. Note that this -# option is superceded by the HAVE_DOT option below. This is only a fallback. It is -# recommended to install and use dot, since it yield more powerful graphs. - -CLASS_DIAGRAMS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = NO - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found on the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_WIDTH = 1024 - -# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_HEIGHT = 1024 - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermedate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO - -# The CGI_NAME tag should be the name of the CGI script that -# starts the search engine (doxysearch) with the correct parameters. -# A script with this name will be generated by doxygen. - -CGI_NAME = search.cgi - -# The CGI_URL tag should be the absolute URL to the directory where the -# cgi binaries are located. See the documentation of your http daemon for -# details. - -CGI_URL = - -# The DOC_URL tag should be the absolute URL to the directory where the -# documentation is located. If left blank the absolute path to the -# documentation, with file:// prepended to it, will be used. - -DOC_URL = - -# The DOC_ABSPATH tag should be the absolute path to the directory where the -# documentation is located. If left blank the directory on the local machine -# will be used. - -DOC_ABSPATH = - -# The BIN_ABSPATH tag must point to the directory where the doxysearch binary -# is installed. - -BIN_ABSPATH = /usr/local/bin/ - -# The EXT_DOC_PATHS tag can be used to specify one or more paths to -# documentation generated for other projects. This allows doxysearch to search -# the documentation for these projects as well. - -EXT_DOC_PATHS = diff --git a/memfilepersistence/src/tests/.gitignore b/memfilepersistence/src/tests/.gitignore deleted file mode 100644 index b8a959a31c5..00000000000 --- a/memfilepersistence/src/tests/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -/.depend -/Makefile -/dirconfig.tmp -/test.vlog -/testfile.0 -/testrunner -/vdsroot -memfilepersistence_testrunner_app diff --git a/memfilepersistence/src/tests/CMakeLists.txt b/memfilepersistence/src/tests/CMakeLists.txt deleted file mode 100644 index 1ffe7f785e9..00000000000 --- a/memfilepersistence/src/tests/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(memfilepersistence_testrunner_app TEST - SOURCES - testrunner.cpp - DEPENDS - memfilepersistence_testconformance - memfilepersistence_testdevices - memfilepersistence_testinit - memfilepersistence_testspi - memfilepersistence_testtools -) - -# TODO: Test with a larger chunk size to parallelize test suite runs -vespa_add_test( - NAME memfilepersistence_testrunner_app - COMMAND memfilepersistence_testrunner_app -) diff --git a/memfilepersistence/src/tests/conformance/.gitignore b/memfilepersistence/src/tests/conformance/.gitignore deleted file mode 100644 index 7e7c0fe7fae..00000000000 --- a/memfilepersistence/src/tests/conformance/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.depend -/Makefile diff --git a/memfilepersistence/src/tests/conformance/CMakeLists.txt b/memfilepersistence/src/tests/conformance/CMakeLists.txt deleted file mode 100644 index 077174191c9..00000000000 --- a/memfilepersistence/src/tests/conformance/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(memfilepersistence_testconformance - SOURCES - memfileconformancetest.cpp - DEPENDS -) diff --git a/memfilepersistence/src/tests/conformance/memfileconformancetest.cpp b/memfilepersistence/src/tests/conformance/memfileconformancetest.cpp deleted file mode 100644 index 1bd9e2f4e9c..00000000000 --- a/memfilepersistence/src/tests/conformance/memfileconformancetest.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include - -#include -LOG_SETUP(".test.conformance"); - -using namespace storage::spi; - -namespace storage { -namespace memfile { - - /* -struct MemFileConformanceTest : public ConformanceTest { - struct Factory : public PersistenceFactory { - - PersistenceSPI::UP getPersistenceImplementation() { - return PersistenceSPI::UP(new MemFilePersistence); - } - }; - - MemFileConformanceTest() - : ConformanceTest(PersistenceFactory::UP(new Factory)) {} - - CPPUNIT_TEST_SUITE(MemFileConformanceTest); - DEFINE_CONFORMANCE_TESTS(); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemFileConformanceTest); -*/ - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/device/.gitignore b/memfilepersistence/src/tests/device/.gitignore deleted file mode 100644 index 7e7c0fe7fae..00000000000 --- a/memfilepersistence/src/tests/device/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.depend -/Makefile diff --git a/memfilepersistence/src/tests/device/CMakeLists.txt b/memfilepersistence/src/tests/device/CMakeLists.txt deleted file mode 100644 index 3b7884772b6..00000000000 --- a/memfilepersistence/src/tests/device/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(memfilepersistence_testdevices - SOURCES - mountpointlisttest.cpp - devicemanagertest.cpp - devicestest.cpp - devicemappertest.cpp - partitionmonitortest.cpp - DEPENDS - memfilepersistence -) diff --git a/memfilepersistence/src/tests/device/devicemanagertest.cpp b/memfilepersistence/src/tests/device/devicemanagertest.cpp deleted file mode 100644 index 0d0b4ad1de3..00000000000 --- a/memfilepersistence/src/tests/device/devicemanagertest.cpp +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include -#include - -namespace storage { - -namespace memfile { - -class DeviceManagerTest : public CppUnit::TestFixture { - CPPUNIT_TEST_SUITE(DeviceManagerTest); - CPPUNIT_TEST(testEventClass); - CPPUNIT_TEST(testEventSending); - CPPUNIT_TEST(testXml); - CPPUNIT_TEST_SUITE_END(); - -public: - void testEventClass(); - void testEventSending(); - void testXml(); - - framework::defaultimplementation::FakeClock _clock; -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(DeviceManagerTest); - -void DeviceManagerTest::testEventClass() -{ - // Test that creation various IO events through common errno errors - // generates understandable errors. - { - IOEvent e(IOEvent::createEventFromErrno(1, ENOTDIR, "/mypath")); - CPPUNIT_ASSERT_EQUAL( - std::string("IOEvent(PATH_FAILURE, Not a directory: /mypath, time 1)"), - e.toString(true)); - CPPUNIT_ASSERT_EQUAL(Device::PATH_FAILURE, e.getState()); - } - { - IOEvent e(IOEvent::createEventFromErrno(2, EACCES, "/mypath")); - CPPUNIT_ASSERT_EQUAL( - std::string("IOEvent(NO_PERMISSION, Permission denied: /mypath, time 2)"), - e.toString(true)); - CPPUNIT_ASSERT_EQUAL(Device::NO_PERMISSION, e.getState()); - } - { - IOEvent e(IOEvent::createEventFromErrno(3, EIO, "/mypath")); - CPPUNIT_ASSERT_EQUAL( - std::string("IOEvent(IO_FAILURE, Input/output error: /mypath, time 3)"), - e.toString(true)); - CPPUNIT_ASSERT_EQUAL(Device::IO_FAILURE, e.getState()); - } - { - IOEvent e( - IOEvent::createEventFromErrno(4, EBADF, "/mypath", VESPA_STRLOC)); - CPPUNIT_ASSERT_PREFIX( - std::string("IOEvent(INTERNAL_FAILURE, Bad file descriptor: /mypath" - ", testEventClass in"), - e.toString(true)); - CPPUNIT_ASSERT_EQUAL(Device::INTERNAL_FAILURE, e.getState()); - } -} - -namespace { - - struct Listener : public IOEventListener { - std::ostringstream ost; - - Listener() : ost() { ost << "\n"; } - ~Listener() {} - - void handleDirectoryEvent(Directory& dir, const IOEvent& e) override { - ost << "Dir " << dir.getPath() << ": " << e.toString(true) << "\n"; - } - void handlePartitionEvent(Partition& part, const IOEvent& e) override { - ost << "Partition " << part.getMountPoint() << ": " << e.toString(true) << "\n"; - } - void handleDiskEvent(Disk& disk, const IOEvent& e) override { - ost << "Disk " << disk.getId() << ": " << e.toString(true) << "\n"; - } - }; - -} - -void DeviceManagerTest::testEventSending() -{ - // Test that adding events to directories in the manager actually sends - // these events on to listeners. - DeviceManager manager(DeviceMapper::UP(new SimpleDeviceMapper), _clock); - Listener l; - manager.addIOEventListener(l); - Directory::SP dir(manager.getDirectory("/home/foo/var", 0)); - // IO failures are disk events. Will mark all partitions and - // directories on that disk bad - dir->addEvent(IOEvent::createEventFromErrno(1, EIO, "/home/foo/var/foo")); - dir->addEvent(IOEvent::createEventFromErrno(2, EBADF, "/home/foo/var/bar")); - dir->addEvent(IOEvent::createEventFromErrno(3, EACCES, "/home/foo/var/car")); - dir->addEvent(IOEvent::createEventFromErrno(4, EISDIR, "/home/foo/var/var")); - std::string expected("\n" - "Disk 1: IOEvent(IO_FAILURE, Input/output error: " - "/home/foo/var/foo, time 1)\n" - "Dir /home/foo/var: IOEvent(INTERNAL_FAILURE, Bad file " - "descriptor: /home/foo/var/bar, time 2)\n" - "Dir /home/foo/var: IOEvent(NO_PERMISSION, Permission denied: " - "/home/foo/var/car, time 3)\n" - "Dir /home/foo/var: IOEvent(PATH_FAILURE, Is a directory: " - "/home/foo/var/var, time 4)\n" - ); - CPPUNIT_ASSERT_EQUAL(expected, l.ost.str()); -} - -void DeviceManagerTest::testXml() -{ - DeviceManager manager(DeviceMapper::UP(new SimpleDeviceMapper), _clock); - Directory::SP dir(manager.getDirectory("/home/", 0)); - dir->getPartition().initializeMonitor(); - std::string xml = manager.toXml(" "); - CPPUNIT_ASSERT_MSG(xml, - xml.find("") != std::string::npos); -} - -} - -} diff --git a/memfilepersistence/src/tests/device/devicemappertest.cpp b/memfilepersistence/src/tests/device/devicemappertest.cpp deleted file mode 100644 index a0568268b56..00000000000 --- a/memfilepersistence/src/tests/device/devicemappertest.cpp +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include - -namespace storage { - -namespace memfile { - -class DeviceMapperTest : public CppUnit::TestFixture { - CPPUNIT_TEST_SUITE(DeviceMapperTest); - CPPUNIT_TEST(testSimpleDeviceMapper); - CPPUNIT_TEST(testAdvancedDeviceMapper); - CPPUNIT_TEST_SUITE_END(); - -public: - void testSimpleDeviceMapper(); - void testAdvancedDeviceMapper(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(DeviceMapperTest); - -void DeviceMapperTest::testSimpleDeviceMapper() -{ - SimpleDeviceMapper mapper; - CPPUNIT_ASSERT_EQUAL(uint64_t(1), mapper.getDeviceId("whatever&�")); - CPPUNIT_ASSERT_EQUAL(uint64_t(1), mapper.getDeviceId("whatever&�")); - CPPUNIT_ASSERT_EQUAL(uint64_t(2), mapper.getDeviceId("whatnot")); - std::string expected("Whatever& �=)/%#)="); - CPPUNIT_ASSERT_EQUAL(expected, mapper.getMountPoint(expected)); -} - -void DeviceMapperTest::testAdvancedDeviceMapper() -{ - AdvancedDeviceMapper mapper; - try{ - mapper.getDeviceId("/doesnotexist"); - CPPUNIT_FAIL("Expected exception"); - } catch (vespalib::Exception& e) { - std::string what(e.what()); - CPPUNIT_ASSERT_CONTAIN( - "Failed to run stat to find data on file /doesnotexist", what); - } -} - -} - -} // storage diff --git a/memfilepersistence/src/tests/device/devicestest.cpp b/memfilepersistence/src/tests/device/devicestest.cpp deleted file mode 100644 index bb6ad3ee6a2..00000000000 --- a/memfilepersistence/src/tests/device/devicestest.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include - -namespace storage { - -namespace memfile { - -class DevicesTest : public CppUnit::TestFixture { - CPPUNIT_TEST_SUITE(DevicesTest); - CPPUNIT_TEST(testDisk); - CPPUNIT_TEST(testPartition); - CPPUNIT_TEST(testDirectory); - CPPUNIT_TEST_SUITE_END(); - -public: - void testDisk(); - void testPartition(); - void testDirectory(); - - framework::defaultimplementation::FakeClock _clock; -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(DevicesTest); - -void DevicesTest::testDisk() -{ - DeviceManager manager(DeviceMapper::UP(new SimpleDeviceMapper), _clock); - Disk::SP disk1(manager.getDisk("/something/on/disk")); - Disk::SP disk2(manager.getDisk("/something/on/disk")); - CPPUNIT_ASSERT_EQUAL(disk1->getId(), disk2->getId()); - CPPUNIT_ASSERT_EQUAL(disk1.get(), disk2.get()); - Disk::SP disk3(manager.getDisk("/something/on/disk2")); - CPPUNIT_ASSERT(disk2->getId() != disk3->getId()); - disk3->toString(); // Add code coverage -} - -void DevicesTest::testPartition() -{ - DeviceManager manager(DeviceMapper::UP(new SimpleDeviceMapper), _clock); - Partition::SP part(manager.getPartition("/etc")); - CPPUNIT_ASSERT_EQUAL(std::string("/etc"), part->getMountPoint()); - part->toString(); // Add code coverage -} - -void DevicesTest::testDirectory() -{ - DeviceManager manager(DeviceMapper::UP(new SimpleDeviceMapper), _clock); - Directory::SP dir1(manager.getDirectory("/on/disk", 0)); - CPPUNIT_ASSERT_EQUAL(std::string("/on/disk"), dir1->getPath()); - CPPUNIT_ASSERT(dir1->getLastEvent() == 0); - CPPUNIT_ASSERT_EQUAL(Device::OK, dir1->getState()); - CPPUNIT_ASSERT(dir1->isOk()); - CPPUNIT_ASSERT_EQUAL(std::string("/on/disk 0"), dir1->toString()); - - dir1->addEvent(Device::IO_FAILURE, "Ouch", ""); - CPPUNIT_ASSERT(!dir1->isOk()); - CPPUNIT_ASSERT(dir1->getLastEvent() != 0); - CPPUNIT_ASSERT_EQUAL(std::string("/on/disk 5 0 Ouch"), dir1->toString()); - dir1->toString(); // Add code coverage -} - -} - -} // storage diff --git a/memfilepersistence/src/tests/device/mountpointlisttest.cpp b/memfilepersistence/src/tests/device/mountpointlisttest.cpp deleted file mode 100644 index 56b59926dff..00000000000 --- a/memfilepersistence/src/tests/device/mountpointlisttest.cpp +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include - -using vespalib::fileExists; -using vespalib::isDirectory; -using vespalib::isSymLink; -using vespalib::readLink; - -namespace storage { - -namespace memfile { - -class MountPointList_Test : public CppUnit::TestFixture { - CPPUNIT_TEST_SUITE(MountPointList_Test); - CPPUNIT_TEST(testScanning); - CPPUNIT_TEST(testStatusFile); - CPPUNIT_TEST(testInitDisks); - CPPUNIT_TEST_SUITE_END(); - - static const std::string _prefix; - -public: - void testScanning(); - void testStatusFile(); - void testInitDisks(); - - void init(); - void tearDown() override; - - framework::defaultimplementation::FakeClock _clock; - -private: - DeviceManager::UP newDeviceManager() { - return DeviceManager::UP( - new DeviceManager( - DeviceMapper::UP(new SimpleDeviceMapper), - _clock)); - } -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MountPointList_Test); - -const std::string MountPointList_Test::_prefix("./vdsroot"); - -namespace { - void run(const std::string& cmd) { - CPPUNIT_ASSERT_MESSAGE(cmd, system(cmd.c_str()) == 0); - } -} - -void MountPointList_Test::init() -{ - tearDown(); - run("rm -rf "+_prefix); - run("mkdir -p "+_prefix+"/disks"); - - run("mkdir "+_prefix+"/disks/d0"); // Regular dir - // disks/d1 intentionally missing - run("mkdir "+_prefix+"/disks/D2"); // Wrongly named dir - run("mkdir "+_prefix+"/disks/d3"); // Regular non-empty dir - run("touch "+_prefix+"/disks/d3/foo"); - run("touch "+_prefix+"/disks/d4"); // Not a dir - run("ln -s D2 "+_prefix+"/disks/d5"); // Symlink to dir - run("ln -s d4 "+_prefix+"/disks/d6"); // Symlink to file -} - -void MountPointList_Test::tearDown() {} - -void MountPointList_Test::testScanning() -{ - init(); - MountPointList list(_prefix, - std::vector(), - DeviceManager::UP( - new DeviceManager( - DeviceMapper::UP(new SimpleDeviceMapper), - _clock))); - list.scanForDisks(); - - // Check that we got the expected entries. - CPPUNIT_ASSERT_EQUAL(7u, list.getSize()); - - for (uint32_t i=0; i<7u; ++i) { - std::ostringstream ost; - ost << _prefix << "/disks/d" << i; - CPPUNIT_ASSERT_EQUAL(ost.str(), list[i].getPath()); - } - - // Note.. scanForDisks() should not in any circumstances access the - // disks. Thus it should not know that d1 is inaccessible, or that d6 - // is actually a symlink to a file - CPPUNIT_ASSERT_EQUAL(Device::OK, list[0].getState()); - CPPUNIT_ASSERT_EQUAL(Device::NOT_FOUND, list[1].getState()); - CPPUNIT_ASSERT_EQUAL(Device::NOT_FOUND, list[2].getState()); - CPPUNIT_ASSERT_EQUAL(Device::OK, list[3].getState()); - CPPUNIT_ASSERT_EQUAL(Device::PATH_FAILURE, list[4].getState()); - CPPUNIT_ASSERT_EQUAL(Device::OK, list[5].getState()); - CPPUNIT_ASSERT_EQUAL(Device::OK, list[6].getState()); - - list.verifyHealthyDisks(-1); - CPPUNIT_ASSERT_EQUAL(Device::OK, list[0].getState()); - CPPUNIT_ASSERT_EQUAL(Device::NOT_FOUND, list[1].getState()); - CPPUNIT_ASSERT_EQUAL(Device::NOT_FOUND, list[2].getState()); - CPPUNIT_ASSERT_EQUAL(Device::INTERNAL_FAILURE, list[3].getState()); - CPPUNIT_ASSERT_EQUAL(Device::PATH_FAILURE, list[4].getState()); - CPPUNIT_ASSERT_EQUAL(Device::OK, list[5].getState()); - CPPUNIT_ASSERT_EQUAL(Device::PATH_FAILURE, list[6].getState()); -} - -void MountPointList_Test::testStatusFile() -{ - init(); - std::string statusFileName(_prefix + "/disks.status"); - - // Try reading non-existing file, and writing a file - { - MountPointList list(_prefix, - std::vector(), - DeviceManager::UP( - new DeviceManager( - DeviceMapper::UP(new SimpleDeviceMapper), - _clock))); - - _clock.setAbsoluteTimeInSeconds(5678); - list.scanForDisks(); - - // File does not currently exist, that should be ok though. - list.readFromFile(); - list.verifyHealthyDisks(-1); - CPPUNIT_ASSERT_EQUAL(7u, list.getSize()); - list[5].addEvent(IOEvent(1234, Device::IO_FAILURE, "Argh", "Hmm")); - CPPUNIT_ASSERT_EQUAL(Device::IO_FAILURE, list[5].getState()); - - // Write to file. - list.writeToFile(); - } - - // Check contents of file. - { - std::ifstream in(statusFileName.c_str()); - std::string line; - CPPUNIT_ASSERT(std::getline(in, line)); - - CPPUNIT_ASSERT_PREFIX( - std::string(_prefix + "/disks/d1 1 5678 Disk not found " - "during scanning of disks directory"), - line); - CPPUNIT_ASSERT(std::getline(in, line)); - CPPUNIT_ASSERT_PREFIX( - std::string(_prefix +"/disks/d2 1 5678 Disk not found during scanning of " - "disks directory"), - line); - CPPUNIT_ASSERT(std::getline(in, line)); - CPPUNIT_ASSERT_PREFIX( - std::string(_prefix + "/disks/d3 4 5678 Foreign data in mountpoint. New " - "mountpoints added should be empty."), - line); - CPPUNIT_ASSERT(std::getline(in, line)); - CPPUNIT_ASSERT_PREFIX( - std::string(_prefix + "/disks/d4 2 5678 File d4 in disks directory is not " - "a directory."), - line); - CPPUNIT_ASSERT(std::getline(in, line)); - CPPUNIT_ASSERT_PREFIX(std::string(_prefix + "/disks/d5 5 1234 Argh"), - line); - CPPUNIT_ASSERT(std::getline(in, line)); - CPPUNIT_ASSERT_PREFIX( - std::string(_prefix + "/disks/d6 2 5678 The path exist, but is not a " - "directory."), - line); - CPPUNIT_ASSERT(std::getline(in, line)); - CPPUNIT_ASSERT_EQUAL(std::string("EOF"), line); - } - - // Starting over to get new device instances. - // Scan disk, read file, and check that erronious disks are not used. - { - MountPointList list(_prefix, - std::vector(), - DeviceManager::UP( - new DeviceManager( - DeviceMapper::UP(new SimpleDeviceMapper), - _clock))); - list.scanForDisks(); - list.readFromFile(); - // Check that we got the expected entries. - CPPUNIT_ASSERT_EQUAL(7u, list.getSize()); - - // Note.. scanForDisks() should not under any circumstance access the - // disks. Thus it should not know that d1 is inaccessible. - CPPUNIT_ASSERT_EQUAL(Device::OK, list[0].getState()); - CPPUNIT_ASSERT_EQUAL(Device::NOT_FOUND, list[1].getState()); - CPPUNIT_ASSERT_EQUAL(Device::NOT_FOUND, list[2].getState()); - CPPUNIT_ASSERT_EQUAL(Device::INTERNAL_FAILURE, list[3].getState()); - CPPUNIT_ASSERT_EQUAL(Device::PATH_FAILURE, list[4].getState()); - CPPUNIT_ASSERT_EQUAL(Device::IO_FAILURE, list[5].getState()); - CPPUNIT_ASSERT_EQUAL(Device::PATH_FAILURE, list[6].getState()); - } -} - -void MountPointList_Test::testInitDisks() -{ - vespalib::string d3target = "d3target"; - vespalib::string foodev = _prefix + "/foodev"; - vespalib::string bardev = _prefix + "/bardev"; - - tearDown(); - run("rm -rf " + _prefix); - run("mkdir -p " + _prefix + "/disks/d2"); - run("ln -s " + d3target + " " + _prefix + "/disks/d3"); - - std::vector diskPaths { - // disks/d0 should become a regular directory - _prefix + "/disks/d0", - // disks/d1 should be a symlink to /foo - foodev, - // disks/d2 should already be a directory - "/ignored", - // disks/d3 should already be a symlink - "/ignored2" - }; - - MountPointList list(_prefix, diskPaths, newDeviceManager()); - list.initDisks(); - - CPPUNIT_ASSERT(isDirectory(_prefix + "/disks")); - CPPUNIT_ASSERT(isDirectory(_prefix + "/disks/d0")); - CPPUNIT_ASSERT(isSymLink(_prefix + "/disks/d1")); - CPPUNIT_ASSERT_EQUAL(foodev, readLink(_prefix + "/disks/d1")); - CPPUNIT_ASSERT(isDirectory(_prefix + "/disks/d2")); - CPPUNIT_ASSERT(isSymLink(_prefix + "/disks/d3")); - CPPUNIT_ASSERT_EQUAL(d3target, readLink(_prefix + "/disks/d3")); -} - -} // memfile - -} // storage - diff --git a/memfilepersistence/src/tests/device/partitionmonitortest.cpp b/memfilepersistence/src/tests/device/partitionmonitortest.cpp deleted file mode 100644 index a31f1fde28b..00000000000 --- a/memfilepersistence/src/tests/device/partitionmonitortest.cpp +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include - -namespace storage { - -namespace memfile { - -struct PartitionMonitorTest : public CppUnit::TestFixture -{ - void testNormalUsage(); - void testHighInodeFillrate(); - void testAlwaysStatPolicy(); - void testPeriodPolicy(); - void testStatOncePolicy(); - void testDynamicPolicy(); - void testIsFull(); - - CPPUNIT_TEST_SUITE(PartitionMonitorTest); - CPPUNIT_TEST(testNormalUsage); - CPPUNIT_TEST(testHighInodeFillrate); - CPPUNIT_TEST(testAlwaysStatPolicy); - CPPUNIT_TEST(testPeriodPolicy); - CPPUNIT_TEST(testStatOncePolicy); - CPPUNIT_TEST(testDynamicPolicy); - CPPUNIT_TEST(testIsFull); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(PartitionMonitorTest); - -struct FakeStatter : public PartitionMonitor::Statter { - struct statvfs _info; - - FakeStatter() { - _info.f_bsize = 4096; - _info.f_frsize = 4096; - _info.f_blocks = 1000; - _info.f_bfree = 500; - _info.f_bavail = 400; - _info.f_files = 64; - _info.f_ffree = 32; - _info.f_favail = 30; - _info.f_fsid = 13; - _info.f_namemax = 256; - } - void removeData(uint32_t size) { - _info.f_bavail += (size / _info.f_bsize); - _info.f_bfree += (size / _info.f_bsize); - } - void addData(uint32_t size) { - _info.f_bavail -= (size / _info.f_bsize); - _info.f_bfree -= (size / _info.f_bsize); - } - - void statFileSystem(const std::string&, struct statvfs& info) override { - info = _info; - } -}; - -void PartitionMonitorTest::testNormalUsage() -{ - const std::string file_name = TEST_PATH("testrunner.cpp"); - PartitionMonitor monitor(file_name); - FakeStatter* statter = new FakeStatter(); - monitor.setStatter(std::unique_ptr(statter)); - std::string expected( - "PartitionMonitor(" + file_name + ", STAT_PERIOD(100), " - "2048000/3686400 used - 55.5556 % full)"); - CPPUNIT_ASSERT_EQUAL(expected, monitor.toString(false)); - expected = - "PartitionMonitor(" + file_name + ") {\n" - " Fill rate: 55.5556 %\n" - " Inode fill rate: 51.6129 %\n" - " Detected block size: 4096\n" - " File system id: 13\n" - " Total size: 3686400 (3600 kB)\n" - " Used size: 2048000 (2000 kB)\n" - " Queries since last stat: 0\n" - " Monitor policy: STAT_PERIOD(100)\n" - " Root only ratio 0\n" - " Max fill rate 98 %\n" - "}"; - CPPUNIT_ASSERT_EQUAL(expected, monitor.toString(true)); - CPPUNIT_ASSERT(monitor.getFillRate() > 0.55); -} - -void PartitionMonitorTest::testHighInodeFillrate() -{ - const std::string file_name = TEST_PATH("testrunner.cpp"); - PartitionMonitor monitor(file_name); - FakeStatter* statter = new FakeStatter(); - statter->_info.f_favail = 2; - monitor.setStatter(std::unique_ptr(statter)); - std::string expected( - "PartitionMonitor(" + file_name + ", STAT_PERIOD(100), " - "2048000/3686400 used - 94.1176 % full (inodes))"); - CPPUNIT_ASSERT_EQUAL(expected, monitor.toString(false)); - expected = - "PartitionMonitor(" + file_name + ") {\n" - " Fill rate: 55.5556 %\n" - " Inode fill rate: 94.1176 %\n" - " Detected block size: 4096\n" - " File system id: 13\n" - " Total size: 3686400 (3600 kB)\n" - " Used size: 2048000 (2000 kB)\n" - " Queries since last stat: 0\n" - " Monitor policy: STAT_PERIOD(100)\n" - " Root only ratio 0\n" - " Max fill rate 98 %\n" - "}"; - CPPUNIT_ASSERT_EQUAL(expected, monitor.toString(true)); - CPPUNIT_ASSERT(monitor.getFillRate() > 0.94); -} - -void PartitionMonitorTest::testAlwaysStatPolicy() -{ - PartitionMonitor monitor(TEST_PATH("testrunner.cpp")); - FakeStatter* statter = new FakeStatter(); - monitor.setStatter(std::unique_ptr(statter)); - monitor.setAlwaysStatPolicy(); - for (uint32_t i=0; i<10; ++i) { - monitor.getFillRate(); - CPPUNIT_ASSERT_EQUAL(0u, monitor._queriesSinceStat); - } -} - -void PartitionMonitorTest::testPeriodPolicy() -{ - PartitionMonitor monitor(TEST_PATH("testrunner.cpp")); - FakeStatter* statter = new FakeStatter(); - monitor.setStatter(std::unique_ptr(statter)); - monitor.setStatPeriodPolicy(4); - for (uint32_t i=1; i<16; ++i) { - monitor.getFillRate(); - CPPUNIT_ASSERT_EQUAL(i % 4, monitor._queriesSinceStat); - } -} - -void PartitionMonitorTest::testStatOncePolicy() -{ - PartitionMonitor monitor(TEST_PATH("testrunner.cpp")); - FakeStatter* statter = new FakeStatter(); - monitor.setStatter(std::unique_ptr(statter)); - monitor.setStatOncePolicy(); - for (uint32_t i=1; i<16; ++i) { - monitor.getFillRate(); - CPPUNIT_ASSERT_EQUAL(i, monitor._queriesSinceStat); - } -} - -void PartitionMonitorTest::testDynamicPolicy() -{ - PartitionMonitor monitor(TEST_PATH("testrunner.cpp")); - FakeStatter* statter = new FakeStatter(); - monitor.setStatter(std::unique_ptr(statter)); - monitor.setStatDynamicPolicy(2); - // Add some data, such that we see that period goes down - CPPUNIT_ASSERT_EQUAL(uint64_t(3698), monitor.calcDynamicPeriod()); - CPPUNIT_ASSERT_EQUAL(55, (int) (100 * monitor.getFillRate())); - monitor.addingData(256 * 1024); - CPPUNIT_ASSERT_EQUAL(uint64_t(2592), monitor.calcDynamicPeriod()); - CPPUNIT_ASSERT_EQUAL(62, (int) (100 * monitor.getFillRate())); - monitor.addingData(512 * 1024); - CPPUNIT_ASSERT_EQUAL(uint64_t(968), monitor.calcDynamicPeriod()); - CPPUNIT_ASSERT_EQUAL(76, (int) (100 * monitor.getFillRate())); - // Add such that we hint that we have more data than possible on disk - monitor.addingData(1024 * 1024); - // Let fake stat just have a bit more data than before - statter->addData(256 * 1024); - // With high fill rate, we should check stat each time - CPPUNIT_ASSERT_EQUAL(uint64_t(1), monitor.calcDynamicPeriod()); - // As period is 1, we will now do a new stat, it should find we - // actually have less fill rate - CPPUNIT_ASSERT_EQUAL(62, (int) (100 * monitor.getFillRate())); -} - -void PartitionMonitorTest::testIsFull() -{ - PartitionMonitor monitor(TEST_PATH("testrunner.cpp")); - monitor.setMaxFillness(0.85); - FakeStatter* statter = new FakeStatter(); - monitor.setStatOncePolicy(); - monitor.setStatter(std::unique_ptr(statter)); - - CPPUNIT_ASSERT_EQUAL(55, (int) (100 * monitor.getFillRate())); - CPPUNIT_ASSERT(!monitor.isFull()); - monitor.addingData(512 * 1024); - CPPUNIT_ASSERT_EQUAL(69, (int) (100 * monitor.getFillRate())); - CPPUNIT_ASSERT(!monitor.isFull()); - monitor.addingData(600 * 1024); - CPPUNIT_ASSERT_EQUAL(86, (int) (100 * monitor.getFillRate())); - CPPUNIT_ASSERT(monitor.isFull()); - monitor.removingData(32 * 1024); - CPPUNIT_ASSERT_EQUAL(85, (int) (100 * monitor.getFillRate())); - CPPUNIT_ASSERT(monitor.isFull()); - monitor.removingData(32 * 1024); - CPPUNIT_ASSERT_EQUAL(84, (int) (100 * monitor.getFillRate())); - CPPUNIT_ASSERT(!monitor.isFull()); -} - -} - -} // storage diff --git a/memfilepersistence/src/tests/helper/CMakeLists.txt b/memfilepersistence/src/tests/helper/CMakeLists.txt deleted file mode 100644 index 3a12123250c..00000000000 --- a/memfilepersistence/src/tests/helper/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(memfilepersistence_testhelper - SOURCES - testhelper.cpp - DEPENDS - vdstestlib -) diff --git a/memfilepersistence/src/tests/helper/testhelper.cpp b/memfilepersistence/src/tests/helper/testhelper.cpp deleted file mode 100644 index e6662c52d1b..00000000000 --- a/memfilepersistence/src/tests/helper/testhelper.cpp +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include - -#include -LOG_SETUP(".testhelper"); - -namespace storage { - -void addStorageDistributionConfig(vdstestlib::DirConfig& dc) -{ - vdstestlib::DirConfig::Config* config; - config = &dc.getConfig("stor-distribution", true); - config->clear(); - config->set("group[1]"); - config->set("group[0].name", "foo"); - config->set("group[0].index", "0"); - config->set("group[0].nodes[50]"); - - for (uint32_t i = 0; i < 50; i++) { - std::ostringstream key; key << "group[0].nodes[" << i << "].index"; - std::ostringstream val; val << i; - config->set(key.str(), val.str()); - } -} - -vdstestlib::DirConfig getStandardConfig(bool storagenode) { - vdstestlib::DirConfig dc; - vdstestlib::DirConfig::Config* config; - config = &dc.addConfig("stor-cluster"); - config = &dc.addConfig("load-type"); - config = &dc.addConfig("bucket"); - config = &dc.addConfig("messagebus"); - config = &dc.addConfig("stor-prioritymapping"); - config = &dc.addConfig("stor-bucketdbupdater"); - config = &dc.addConfig("metricsmanager"); - config->set("consumer[1]"); - config->set("consumer[0].name", "\"status\""); - config->set("consumer[0].addedmetrics[1]"); - config->set("consumer[0].addedmetrics[0]", "\"*\""); - config = &dc.addConfig("stor-communicationmanager"); - config->set("rpcport", "0"); - config->set("mbusport", "0"); - config = &dc.addConfig("stor-bucketdb"); - config->set("chunklevel", "0"); - config = &dc.addConfig("stor-distributormanager"); - config = &dc.addConfig("stor-opslogger"); - config = &dc.addConfig("stor-memfilepersistence"); - // Easier to see what goes wrong with only 1 thread per disk. - config->set("minimum_file_meta_slots", "2"); - config->set("minimum_file_header_block_size", "368"); - config->set("minimum_file_size", "4096"); - config->set("threads[1]"); - config->set("threads[0].lowestpri 255"); - config->set("dir_spread", "4"); - config->set("dir_levels", "0"); - // Unit tests typically use fake low time values, so don't complain - // about them or compact/delete them by default. Override in tests testing that - // behavior - config = &dc.addConfig("persistence"); - config->set("keep_remove_time_period", "2000000000"); - config->set("revert_time_period", "2000000000"); - config = &dc.addConfig("stor-bouncer"); - config = &dc.addConfig("stor-integritychecker"); - config = &dc.addConfig("stor-bucketmover"); - config = &dc.addConfig("stor-messageforwarder"); - config = &dc.addConfig("stor-server"); - config->set("enable_dead_lock_detector", "false"); - config->set("enable_dead_lock_detector_warnings", "false"); - config->set("max_merges_per_node", "25"); - config->set("max_merge_queue_size", "20"); - config->set("root_folder", - (storagenode ? "vdsroot" : "vdsroot.distributor")); - config->set("is_distributor", - (storagenode ? "false" : "true")); - config = &dc.addConfig("stor-devices"); - config->set("root_folder", - (storagenode ? "vdsroot" : "vdsroot.distributor")); - config = &dc.addConfig("stor-status"); - config->set("httpport", "0"); - config = &dc.addConfig("stor-visitor"); - config->set("defaultdocblocksize", "8192"); - // By default, need "old" behaviour of maxconcurrent - config->set("maxconcurrentvisitors_fixed", "4"); - config->set("maxconcurrentvisitors_variable", "0"); - config = &dc.addConfig("stor-visitordispatcher"); - addFileConfig(dc, "documenttypes", TEST_PATH("config-doctypes.cfg")); - addStorageDistributionConfig(dc); - return dc; -} - -void addFileConfig(vdstestlib::DirConfig& dc, - const std::string& configDefName, - const std::string& fileName) -{ - vdstestlib::DirConfig::Config* config; - config = &dc.getConfig(configDefName, true); - config->clear(); - std::ifstream in(fileName.c_str()); - std::string line; - while (std::getline(in, line, '\n')) { - std::string::size_type pos = line.find(' '); - if (pos == std::string::npos) { - config->set(line); - } else { - config->set(line.substr(0, pos), line.substr(pos + 1)); - } - } - in.close(); -} - -TestName::TestName(const std::string& n) - : name(n) -{ - LOG(debug, "Starting test %s", name.c_str()); -} - -TestName::~TestName() { - LOG(debug, "Done with test %s", name.c_str()); -} - -} // storage diff --git a/memfilepersistence/src/tests/helper/testhelper.h b/memfilepersistence/src/tests/helper/testhelper.h deleted file mode 100644 index b8dcc82140a..00000000000 --- a/memfilepersistence/src/tests/helper/testhelper.h +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once -#include -#include - - -#include -#include - -#define ASSERT_REPLY_COUNT(count, dummylink) \ - { \ - std::ostringstream msgost; \ - if ((dummylink).getNumReplies() != count) { \ - for (uint32_t ijx=0; ijx<(dummylink).getNumReplies(); ++ijx) { \ - msgost << (dummylink).getReply(ijx)->toString(true) << "\n"; \ - } \ - } \ - CPPUNIT_ASSERT_EQUAL_MSG(msgost.str(), size_t(count), \ - (dummylink).getNumReplies()); \ - } -#define ASSERT_COMMAND_COUNT(count, dummylink) \ - { \ - std::ostringstream msgost; \ - if ((dummylink).getNumCommands() != count) { \ - for (uint32_t ijx=0; ijx<(dummylink).getNumCommands(); ++ijx) { \ - msgost << (dummylink).getCommand(ijx)->toString(true) << "\n"; \ - } \ - } \ - CPPUNIT_ASSERT_EQUAL_MSG(msgost.str(), size_t(count), \ - (dummylink).getNumCommands()); \ - } - -namespace storage { - -void addFileConfig(vdstestlib::DirConfig& dc, - const std::string& configDefName, - const std::string& fileName); - - -void addStorageDistributionConfig(vdstestlib::DirConfig& dc); - -vdstestlib::DirConfig getStandardConfig(bool storagenode); - -// Class used to print start and end of test. Enable debug when you want to see -// which test creates what output or where we get stuck -struct TestName { - std::string name; - TestName(const std::string& n); - ~TestName(); -}; - -} // storage - diff --git a/memfilepersistence/src/tests/init/.gitignore b/memfilepersistence/src/tests/init/.gitignore deleted file mode 100644 index 7e7c0fe7fae..00000000000 --- a/memfilepersistence/src/tests/init/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.depend -/Makefile diff --git a/memfilepersistence/src/tests/init/CMakeLists.txt b/memfilepersistence/src/tests/init/CMakeLists.txt deleted file mode 100644 index de3464a0820..00000000000 --- a/memfilepersistence/src/tests/init/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(memfilepersistence_testinit - SOURCES - filescannertest.cpp - DEPENDS - memfilepersistence -) diff --git a/memfilepersistence/src/tests/init/filescannertest.cpp b/memfilepersistence/src/tests/init/filescannertest.cpp deleted file mode 100644 index 97b7e7ad0c8..00000000000 --- a/memfilepersistence/src/tests/init/filescannertest.cpp +++ /dev/null @@ -1,491 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace storage { -namespace memfile { - -struct FileScannerTest : public CppUnit::TestFixture { - struct TestParameters { - uint32_t filesPerDisk; - uint32_t diskCount; - uint32_t bucketSplitBits; - uint32_t dirLevels; - uint32_t dirSpread; - uint32_t parts; - std::set disksDown; - bool diskDownWithBrokenSymlink; - bool bucketWrongDir; - bool bucketMultipleDirs; - bool bucketMultipleDisks; - bool addTemporaryFiles; - bool addAlienFiles; - bool dirWithNoListPermission; - bool dirWithNoWritePermission; - bool dirWithNoExecutePermission; - bool fileWithNoReadPermission; - bool fileWithNoWritePermission; - - TestParameters() - : filesPerDisk(10), diskCount(5), bucketSplitBits(20), - dirLevels(1), dirSpread(16), parts(1), disksDown(), - diskDownWithBrokenSymlink(false), - bucketWrongDir(false), bucketMultipleDirs(false), - bucketMultipleDisks(false), - addTemporaryFiles(false), addAlienFiles(false), - dirWithNoListPermission(false), - dirWithNoWritePermission(false), - dirWithNoExecutePermission(false), - fileWithNoReadPermission(false), - fileWithNoWritePermission(false) {} - void addAllComplexities() { - disksDown.insert(0); - disksDown.insert(2); - disksDown.insert(4); - bucketWrongDir = true; - bucketMultipleDirs = true; - bucketMultipleDisks = true; - parts = 7; - addTemporaryFiles = true; - addAlienFiles = true; - dirWithNoWritePermission = true; - fileWithNoWritePermission = true; - fileWithNoReadPermission = true; - } - }; - - void testNormalUsage() { - TestParameters params; - runTest(params); - } - void testMultipleParts() { - TestParameters params; - params.parts = 3; - runTest(params); - } - void testBucketInWrongDirectory() { - TestParameters params; - params.bucketWrongDir = true; - runTest(params); - } - void testBucketInMultipleDirectories() { - TestParameters params; - params.bucketMultipleDirs = true; - runTest(params); - } - void testZeroDirLevel() { - TestParameters params; - params.dirLevels = 0; - runTest(params); - } - void testSeveralDirLevels() { - TestParameters params; - params.dirLevels = 3; - runTest(params); - } - void testNonStandardDirSpread() { - TestParameters params; - params.dirSpread = 63; - runTest(params); - } - void testDiskDown() { - TestParameters params; - params.disksDown.insert(1); - runTest(params); - } - void testDiskDownBrokenSymlink() { - TestParameters params; - params.disksDown.insert(1); - params.disksDown.insert(3); - params.diskDownWithBrokenSymlink = true; - runTest(params); - } - void testRemoveTemporaryFile() { - TestParameters params; - params.addTemporaryFiles = true; - runTest(params); - } - void testAlienFile() { - TestParameters params; - params.addAlienFiles = true; - runTest(params); - } - void testUnlistableDirectory() { - TestParameters params; - params.dirWithNoListPermission = true; - runTest(params); - } - void testDirWithNoWritePermission() { - TestParameters params; - params.dirWithNoWritePermission = true; - runTest(params); - } - void testDirWithNoExecutePermission() { - TestParameters params; - params.dirWithNoWritePermission = true; - runTest(params); - } - void testFileWithNoReadPermission() { - TestParameters params; - params.bucketWrongDir = true; - params.fileWithNoReadPermission = true; - runTest(params); - } - void testFileWithNoWritePermission() { - TestParameters params; - params.bucketWrongDir = true; - params.fileWithNoWritePermission = true; - runTest(params); - } - void testAllFailuresCombined() { - TestParameters params; - params.addAllComplexities(); - runTest(params); - } - - CPPUNIT_TEST_SUITE(FileScannerTest); - CPPUNIT_TEST(testNormalUsage); - CPPUNIT_TEST(testMultipleParts); - CPPUNIT_TEST(testBucketInWrongDirectory); - CPPUNIT_TEST(testBucketInMultipleDirectories); - CPPUNIT_TEST(testZeroDirLevel); - CPPUNIT_TEST(testSeveralDirLevels); - CPPUNIT_TEST(testNonStandardDirSpread); - CPPUNIT_TEST(testDiskDown); - CPPUNIT_TEST(testDiskDownBrokenSymlink); - CPPUNIT_TEST(testRemoveTemporaryFile); - CPPUNIT_TEST(testAlienFile); - CPPUNIT_TEST(testUnlistableDirectory); - CPPUNIT_TEST(testDirWithNoWritePermission); - CPPUNIT_TEST(testDirWithNoExecutePermission); - CPPUNIT_TEST(testFileWithNoReadPermission); - CPPUNIT_TEST(testFileWithNoWritePermission); - CPPUNIT_TEST(testAllFailuresCombined); - CPPUNIT_TEST_SUITE_END(); - - // Actual implementation of the tests. - - /** Run a console command and fail test if it fails. */ - void run(std::string cmd); - - /** Struct containing metadata for a single bucket. */ - struct BucketData { - document::BucketId bucket; - uint32_t disk; - std::vector directory; - bool shouldExist; // Set to false for buckets that won't exist due to - // some failure. - - BucketData() : shouldExist(true) {} - - bool sameDir(BucketData& other) const { - return (disk == other.disk && directory == other.directory); - } - }; - - /** - * Create an overview of the buckets we're gonna use in the test. - * (Without any failures introduced) - */ - std::vector createBuckets(const TestParameters& params); - - /** - * Create the data in the bucket map and introduce the failures specified - * in the test. Mark buckets in bucket list that won't exist due to the - * failures so we know how to verify result of test. - */ - void createData(const TestParameters&, std::vector& buckets, - std::vector& tempFiles, - std::vector& alienFiles); - - /** - * Run a test with a given set of parameters, calling createData to set up - * the data, and then using a file scanner to actually list the files. - */ - void runTest(const TestParameters&); - -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(FileScannerTest); - -void -FileScannerTest::run(std::string cmd) -{ - int result = system(cmd.c_str()); - if (result != 0) { - CPPUNIT_FAIL("Failed to run command '" + cmd + "'."); - } -} - -std::vector -FileScannerTest::createBuckets(const TestParameters& params) -{ - std::vector buckets; - BucketDirectoryMapper dirMapper(params.dirLevels, params.dirSpread); - for (uint32_t i=0; i& buckets, - std::vector& tempFiles, - std::vector& alienFiles) -{ - if (params.bucketWrongDir) { - CPPUNIT_ASSERT(params.dirLevels > 0); - buckets[0].directory[0] = (buckets[0].directory[0] + 1) - % params.dirSpread; - } - if (params.bucketMultipleDirs) { - CPPUNIT_ASSERT(params.dirLevels > 0); - BucketData copy(buckets[1]); - copy.directory[0] = (buckets[1].directory[0] + 1) % params.dirSpread; - buckets.push_back(copy); - } - if (params.bucketMultipleDisks && params.dirLevels > 0) { - BucketData copy(buckets[2]); - uint32_t disk = 0; - for (; disk buckets(createBuckets(params)); - std::vector tempFiles; - std::vector alienFiles; - createData(params, buckets, tempFiles, alienFiles); - - framework::defaultimplementation::RealClock clock; - framework::defaultimplementation::ComponentRegisterImpl compReg; - compReg.setClock(clock); - - MountPointList mountPoints("./vdsroot", - std::vector(), - DeviceManager::UP( - new DeviceManager( - DeviceMapper::UP(new SimpleDeviceMapper), - clock))); - mountPoints.init(params.diskCount); - - FileScanner scanner(compReg, mountPoints, - params.dirLevels, params.dirSpread); - std::map foundBuckets; - uint32_t extraBucketsSameDisk = 0; - uint32_t extraBucketsOtherDisk = 0; - for (uint32_t j=0; j notFound; - std::vector wasFound; - std::vector foundNonExisting; - // Verify that found buckets match buckets expected. - for (uint32_t i=0; i::iterator found( - foundBuckets.find(buckets[i].bucket)); - if (buckets[i].shouldExist && found == foundBuckets.end()) { - notFound.push_back(buckets[i]); - } else if (!buckets[i].shouldExist && found != foundBuckets.end()) { - wasFound.push_back(buckets[i]); - } - if (found != foundBuckets.end()) { found->second._checked = true; } - } - for (std::map::iterator it - = foundBuckets.begin(); it != foundBuckets.end(); ++it) - { - if (!it->second._checked) { - foundNonExisting.push_back(it->second); - } - } - if (params.dirWithNoListPermission) { - CPPUNIT_ASSERT(!notFound.empty()); - } else if (!notFound.empty()) { - std::ostringstream ost; - ost << "Failed to find " << notFound.size() << " of " - << buckets.size() << " buckets. Including buckets:"; - for (uint32_t i=0; i<5 && i -#include -#include -#include -#include - -using storage::spi::test::makeSpiBucket; - -namespace storage { -namespace memfile { -namespace { - spi::LoadType defaultLoadType(0, "default"); -} - -class BasicOperationHandlerTest : public SingleDiskMemFileTestUtils -{ - CPPUNIT_TEST_SUITE(BasicOperationHandlerTest); - CPPUNIT_TEST(testGetHeaderOnly); - CPPUNIT_TEST(testGetFieldFiltering); - CPPUNIT_TEST(testRemove); - CPPUNIT_TEST(testRemoveWithNonMatchingTimestamp); - CPPUNIT_TEST(testRemoveWithNonMatchingTimestampAlwaysPersist); - CPPUNIT_TEST(testRemoveForExistingRemoveSameTimestamp); - CPPUNIT_TEST(testRemoveForExistingRemoveNewTimestamp); - CPPUNIT_TEST(testRemoveForExistingRemoveNewTimestampAlwaysPersist); - CPPUNIT_TEST(testRemoveDocumentNotFound); - CPPUNIT_TEST(testRemoveDocumentNotFoundAlwaysPersist); - CPPUNIT_TEST(testRemoveExistingOlderDocumentVersion); - CPPUNIT_TEST(testPutSameTimestampAsRemove); - CPPUNIT_TEST(testUpdateBody); - CPPUNIT_TEST(testUpdateHeaderOnly); - CPPUNIT_TEST(testUpdateTimestampExists); - CPPUNIT_TEST(testUpdateForNonExistentDocWillFail); - CPPUNIT_TEST(testUpdateMayCreateDoc); - CPPUNIT_TEST(testRemoveEntry); - CPPUNIT_TEST(testEraseFromCacheOnFlushException); - CPPUNIT_TEST(testEraseFromCacheOnMaintainException); - CPPUNIT_TEST(testEraseFromCacheOnDeleteBucketException); - CPPUNIT_TEST(list_buckets_returns_empty_set_for_non_default_bucketspace); - CPPUNIT_TEST(get_modified_buckets_returns_empty_set_for_non_default_bucketspace); - CPPUNIT_TEST_SUITE_END(); - - void doTestRemoveDocumentNotFound( - OperationHandler::RemoveType persistRemove); - void doTestRemoveWithNonMatchingTimestamp( - OperationHandler::RemoveType persistRemove); - void doTestRemoveForExistingRemoveNewTimestamp( - OperationHandler::RemoveType persistRemove); -public: - void setupTestConfig(); - void testPutHeadersOnly(); - void testPutHeadersOnlyDocumentNotFound(); - void testPutHeadersOnlyTimestampNotFound(); - void testGetHeaderOnly(); - void testGetFieldFiltering(); - void testRemove(); - void testRemoveWithNonMatchingTimestamp(); - void testRemoveWithNonMatchingTimestampAlwaysPersist(); - void testRemoveForExistingRemoveSameTimestamp(); - void testRemoveForExistingRemoveNewTimestamp(); - void testRemoveForExistingRemoveNewTimestampAlwaysPersist(); - void testRemoveDocumentNotFound(); - void testRemoveDocumentNotFoundAlwaysPersist(); - void testRemoveExistingOlderDocumentVersion(); - void testPutSameTimestampAsRemove(); - void testUpdateBody(); - void testUpdateHeaderOnly(); - void testUpdateTimestampExists(); - void testUpdateForNonExistentDocWillFail(); - void testUpdateMayCreateDoc(); - void testRemoveEntry(); - void testEraseFromCacheOnFlushException(); - void testEraseFromCacheOnMaintainException(); - void testEraseFromCacheOnDeleteBucketException(); - void list_buckets_returns_empty_set_for_non_default_bucketspace(); - void get_modified_buckets_returns_empty_set_for_non_default_bucketspace(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(BasicOperationHandlerTest); - -/** - * Test that doing a header-only get gives back a document containing - * only the document header - */ -void -BasicOperationHandlerTest::testGetHeaderOnly() -{ - document::BucketId bucketId(16, 4); - - Document::SP doc(createRandomDocumentAtLocation(4)); - doc->setValue(doc->getField("hstringval"), document::StringFieldValue("hypnotoad")); - doc->setValue(doc->getField("headerval"), document::IntFieldValue(42)); - - doPut(doc, bucketId, Timestamp(4567), 0); - flush(bucketId); - - spi::GetResult reply = doGet(bucketId, doc->getId(), document::HeaderFields()); - - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, reply.getErrorCode()); - CPPUNIT_ASSERT(reply.hasDocument()); - CPPUNIT_ASSERT_EQUAL(std::string("headerval: 42\nhstringval: hypnotoad\n"), - stringifyFields(reply.getDocument())); - CPPUNIT_ASSERT_EQUAL( - size_t(1), - getPersistenceProvider().getMetrics().headerOnlyGets.getValue()); -} - -void -BasicOperationHandlerTest::testGetFieldFiltering() -{ - document::BucketId bucketId(16, 4); - Document::SP doc(createRandomDocumentAtLocation(4)); - doc->setValue(doc->getField("headerval"), document::IntFieldValue(42)); - doc->setValue(doc->getField("hstringval"), - document::StringFieldValue("groovy")); - - document::FieldSetRepo repo; - - doPut(doc, bucketId, Timestamp(4567), 0); - flush(bucketId); - spi::GetResult reply(doGet(bucketId, - doc->getId(), - *repo.parse(*getTypeRepo(), "testdoctype1:hstringval"))); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, reply.getErrorCode()); - CPPUNIT_ASSERT(reply.hasDocument()); - CPPUNIT_ASSERT_EQUAL(std::string("hstringval: groovy\n"), - stringifyFields(reply.getDocument())); - CPPUNIT_ASSERT_EQUAL( - size_t(1), - getPersistenceProvider().getMetrics().headerOnlyGets.getValue()); -} - -void -BasicOperationHandlerTest::testRemove() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - - document::Document::SP doc = doPut(4, Timestamp(1)); - - CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId, - doc->getId(), - Timestamp(2), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - env()._cache.clear(); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(*doc, *file->getDocument((*file)[0], ALL)); - - CPPUNIT_ASSERT_EQUAL(Timestamp(2), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT((*file)[1].deleted()); - CPPUNIT_ASSERT_EQUAL(DataLocation(0, 0), (*file)[1].getLocation(BODY)); - CPPUNIT_ASSERT_EQUAL((*file)[0].getLocation(HEADER), - (*file)[1].getLocation(HEADER)); -} - -/** - * Test that removing a document with a max timestamp for which there - * is no matching document does not add a remove slot to the memfile - */ -void -BasicOperationHandlerTest::doTestRemoveWithNonMatchingTimestamp( - OperationHandler::RemoveType persistRemove) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::Document::SP doc = doPut(4, Timestamp(1234)); - - CPPUNIT_ASSERT_EQUAL(false, doRemove(bucketId, - doc->getId(), - Timestamp(1233), - persistRemove)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL( - uint32_t(persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE - ? 2 : 1), - file->getSlotCount()); - - int i = 0; - if (persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE) { - CPPUNIT_ASSERT_EQUAL(Timestamp(1233), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT((*file)[0].deleted()); - CPPUNIT_ASSERT_EQUAL(DataLocation(0, 0), (*file)[0].getLocation(BODY)); - CPPUNIT_ASSERT((*file)[0].getLocation(HEADER) - != (*file)[1].getLocation(HEADER)); - CPPUNIT_ASSERT_EQUAL(doc->getId(), file->getDocumentId((*file)[0])); - ++i; - } - - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[i].getTimestamp()); - CPPUNIT_ASSERT(!(*file)[i].deleted()); - CPPUNIT_ASSERT(file->getDocument((*file)[i], ALL)->getValue("content").get()); -} - -/** - * Test that removing a document with a max timestamp for which there - * is no matching document does not add a remove slot to the memfile - */ -void -BasicOperationHandlerTest::testRemoveWithNonMatchingTimestamp() -{ - doTestRemoveWithNonMatchingTimestamp( - OperationHandler::PERSIST_REMOVE_IF_FOUND); -} - -void -BasicOperationHandlerTest::testRemoveWithNonMatchingTimestampAlwaysPersist() -{ - doTestRemoveWithNonMatchingTimestamp( - OperationHandler::ALWAYS_PERSIST_REMOVE); -} - -/** - * Test that doing a remove with a timestamp for which there already - * exists a remove does not add another remove slot - */ -void -BasicOperationHandlerTest::testRemoveForExistingRemoveSameTimestamp() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::Document::SP doc = doPut(4, Timestamp(1234)); - - CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId, - doc->getId(), - Timestamp(1235), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - CPPUNIT_ASSERT_EQUAL(false, doRemove(bucketId, - doc->getId(), - Timestamp(1235), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - // Should only be one remove entry still - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("content").get()); - - CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT((*file)[1].deleted()); -} - -void -BasicOperationHandlerTest::doTestRemoveForExistingRemoveNewTimestamp( - OperationHandler::RemoveType persistRemove) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::Document::SP doc = doPut(4, Timestamp(1234)); - - CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId, - doc->getId(), - Timestamp(1235), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - CPPUNIT_ASSERT_EQUAL(false, doRemove(bucketId, - doc->getId(), - Timestamp(1236), - persistRemove)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL( - uint32_t(persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE - ? 3 : 2), - file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("content").get()); - - CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT((*file)[1].deleted()); - - if (persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE) { - CPPUNIT_ASSERT_EQUAL(Timestamp(1236), (*file)[2].getTimestamp()); - CPPUNIT_ASSERT((*file)[2].deleted()); - } -} - -/** - * Test that doing a second remove with a newer timestamp does not add - * another remove slot when PERSIST_REMOVE_IF_FOUND is specified - */ -void -BasicOperationHandlerTest::testRemoveForExistingRemoveNewTimestamp() -{ - doTestRemoveForExistingRemoveNewTimestamp( - OperationHandler::PERSIST_REMOVE_IF_FOUND); -} - -void -BasicOperationHandlerTest::testRemoveForExistingRemoveNewTimestampAlwaysPersist() -{ - doTestRemoveForExistingRemoveNewTimestamp( - OperationHandler::ALWAYS_PERSIST_REMOVE); -} - -/** - * Test removing an older version of a document. Older version should be removed - * in-place without attempting to add a new slot (which would fail). - */ -void -BasicOperationHandlerTest::testRemoveExistingOlderDocumentVersion() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::Document::SP doc = doPut(4, Timestamp(1234)); - - CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId, - doc->getId(), - Timestamp(1235), - OperationHandler::ALWAYS_PERSIST_REMOVE)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId, - doc->getId(), - Timestamp(1234), - OperationHandler::ALWAYS_PERSIST_REMOVE)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - // Should now be two remove entries. - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(doc->getId(), file->getDocumentId((*file)[0])); - CPPUNIT_ASSERT((*file)[0].deleted()); - - CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(doc->getId(), file->getDocumentId((*file)[1])); - CPPUNIT_ASSERT((*file)[1].deleted()); -} - -void -BasicOperationHandlerTest::doTestRemoveDocumentNotFound( - OperationHandler::RemoveType persistRemove) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::DocumentId docId("userdoc:test:4:0"); - doPut(4, Timestamp(1234)); - - CPPUNIT_ASSERT_EQUAL(false, - doRemove(bucketId, - docId, - Timestamp(1235), - persistRemove)); - - getPersistenceProvider().flush(makeSpiBucket(bucketId), context); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL( - uint32_t(persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE - ? 2 : 1), - file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - if (persistRemove == OperationHandler::ALWAYS_PERSIST_REMOVE) { - CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT((*file)[1].deleted()); - CPPUNIT_ASSERT_EQUAL(docId, file->getDocumentId((*file)[1])); - } -/* TODO: Test this in service layer tests. - CPPUNIT_ASSERT_EQUAL( - uint64_t(1), - env()._metrics.remove[documentapi::LoadType::DEFAULT].notFound.getValue()); -*/ -} - -/** - * Test that removing a non-existing document when PERSIST_EXISTING_ONLY is - * specified does not add a remove entry - */ -void -BasicOperationHandlerTest::testRemoveDocumentNotFound() -{ - doTestRemoveDocumentNotFound( - OperationHandler::PERSIST_REMOVE_IF_FOUND); -} - -void -BasicOperationHandlerTest::testRemoveDocumentNotFoundAlwaysPersist() -{ - doTestRemoveDocumentNotFound( - OperationHandler::ALWAYS_PERSIST_REMOVE); -} - -void -BasicOperationHandlerTest::testPutSameTimestampAsRemove() -{ - document::BucketId bucketId(16, 4); - - document::Document::SP doc = doPut(4, Timestamp(1234)); - - CPPUNIT_ASSERT_EQUAL(true, doRemove(bucketId, - doc->getId(), - Timestamp(1235), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - - // Flush here to avoid put+remove being thrown away by duplicate timestamp - // exception evicting the cache and unpersisted changes. - flush(bucketId); - - doPut(4, Timestamp(1235)); - flush(bucketId); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("content").get()); - - CPPUNIT_ASSERT_EQUAL(Timestamp(1235), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT((*file)[1].deleted()); -} - -/** - * Test that updating body results in a new memfile slot containing - * an updated document - */ -void -BasicOperationHandlerTest::testUpdateBody() -{ - document::BucketId bucketId(16, 4); - document::StringFieldValue updateValue("foo"); - document::Document::SP doc = doPut(4, Timestamp(1234)); - document::Document originalDoc(*doc); - - document::DocumentUpdate::SP update = createBodyUpdate( - doc->getId(), updateValue); - - spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(5678)); - flush(bucketId); - CPPUNIT_ASSERT_EQUAL(1234, (int)result.getExistingTimestamp()); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("content").get()); - CPPUNIT_ASSERT_EQUAL(*(originalDoc.getValue("content")), - *file->getDocument((*file)[0], ALL)->getValue("content")); - - CPPUNIT_ASSERT_EQUAL(Timestamp(5678), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[1], ALL)->getValue("content").get()); - CPPUNIT_ASSERT_EQUAL(updateValue, - dynamic_cast( - *file->getDocument((*file)[1], ALL)->getValue( - "content"))); - CPPUNIT_ASSERT_EQUAL( - size_t(0), - getPersistenceProvider().getMetrics().headerOnlyUpdates.getValue()); -} - -void -BasicOperationHandlerTest::testUpdateHeaderOnly() -{ - document::BucketId bucketId(16, 4); - document::IntFieldValue updateValue(42); - document::Document::SP doc = doPut(4, Timestamp(1234)); - - document::DocumentUpdate::SP update = createHeaderUpdate( - doc->getId(), updateValue); - - spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(5678)); - flush(bucketId); - CPPUNIT_ASSERT_EQUAL(1234, (int)result.getExistingTimestamp()); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1234), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[0], ALL)->getValue("headerval").get() == - NULL); - - CPPUNIT_ASSERT_EQUAL(Timestamp(5678), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT(file->getDocument((*file)[1], ALL)->getValue("headerval").get()); - CPPUNIT_ASSERT_EQUAL(updateValue, - dynamic_cast( - *file->getDocument((*file)[1], ALL)->getValue( - "headerval"))); - CPPUNIT_ASSERT_EQUAL( - size_t(1), - getPersistenceProvider().getMetrics().headerOnlyUpdates.getValue()); -} - -void -BasicOperationHandlerTest::testUpdateTimestampExists() -{ - document::BucketId bucketId(16, 4); - document::IntFieldValue updateValue(42); - document::Document::SP doc = doPut(4, Timestamp(1234)); - - document::DocumentUpdate::SP update = createHeaderUpdate( - doc->getId(), updateValue); - - spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(1234)); - flush(bucketId); - CPPUNIT_ASSERT_EQUAL(spi::Result::TRANSIENT_ERROR, result.getErrorCode()); -} - -void -BasicOperationHandlerTest::testUpdateForNonExistentDocWillFail() -{ - document::BucketId bucketId(16, 4); - document::IntFieldValue updateValue(42); - Timestamp timestamp(5678); - - // Is there an easier way to get a DocumentId? - document::Document::UP doc( - createRandomDocumentAtLocation(4, timestamp.getTime())); - const DocumentId& documentId = doc->getId(); - - document::DocumentUpdate::SP update = createHeaderUpdate( - documentId, updateValue); - - spi::UpdateResult result = doUpdate(bucketId, update, timestamp); - flush(bucketId); - CPPUNIT_ASSERT_EQUAL(0, (int)result.getExistingTimestamp()); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(0), file->getSlotCount()); -} - -void -BasicOperationHandlerTest::testUpdateMayCreateDoc() -{ - document::BucketId bucketId(16, 4); - document::IntFieldValue updateValue(42); - Timestamp timestamp(5678); - - // Is there an easier way to get a DocumentId? - document::Document::UP doc( - createRandomDocumentAtLocation(4, timestamp.getTime())); - const DocumentId& documentId = doc->getId(); - - document::DocumentUpdate::SP update = createHeaderUpdate( - documentId, updateValue); - update->setCreateIfNonExistent(true); - - spi::UpdateResult result = doUpdate(bucketId, update, timestamp); - flush(bucketId); - CPPUNIT_ASSERT_EQUAL(timestamp.getTime(), - (uint64_t)result.getExistingTimestamp()); - - MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(timestamp, (*file)[0].getTimestamp()); - - auto headerval = file->getDocument((*file)[0], ALL)->getValue("headerval"); - CPPUNIT_ASSERT(headerval.get() != nullptr); - CPPUNIT_ASSERT_EQUAL(updateValue, - dynamic_cast(*headerval)); -} - -void -BasicOperationHandlerTest::testRemoveEntry() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - - doPut(4, Timestamp(1234)); - Document::SP doc = doPut(4, Timestamp(2345)); - doPut(4, Timestamp(3456)); - - getPersistenceProvider().removeEntry(makeSpiBucket(bucketId), spi::Timestamp(1234), context); - getPersistenceProvider().removeEntry(makeSpiBucket(bucketId), spi::Timestamp(3456), context); - flush(bucketId); - - memfile::MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(2345), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(*doc, *file->getDocument((*file)[0], ALL)); -} - -void -BasicOperationHandlerTest::setupTestConfig() -{ - using MemFileConfig = vespa::config::storage::StorMemfilepersistenceConfig; - using MemFileConfigBuilder - = vespa::config::storage::StorMemfilepersistenceConfigBuilder; - MemFileConfigBuilder builder( - *env().acquireConfigReadLock().memFilePersistenceConfig()); - builder.minimumFileMetaSlots = 2; - builder.minimumFileHeaderBlockSize = 3000; - auto newConfig = std::unique_ptr(new MemFileConfig(builder)); - env().acquireConfigWriteLock().setMemFilePersistenceConfig( - std::move(newConfig)); -} - -void -BasicOperationHandlerTest::testEraseFromCacheOnFlushException() -{ - document::BucketId bucketId(16, 4); - - setupTestConfig(); - - document::Document::SP doc( - createRandomDocumentAtLocation(4, 2345, 1024, 1024)); - doPut(doc, bucketId, Timestamp(2345)); - flush(bucketId); - // Must throw out cache to re-create lazyfile - env()._cache.clear(); - - env()._lazyFileFactory = - std::unique_ptr( - new SimulatedFailureLazyFile::Factory); - - // Try partial write, followed by full rewrite - for (int i = 0; i < 2; ++i) { - for (int j = 0; j < i+1; ++j) { - document::Document::SP doc2( - createRandomDocumentAtLocation(4, 4000 + j, 1500, 1500)); - doPut(doc2, bucketId, Timestamp(4000 + j)); - } - spi::Result result = flush(bucketId); - CPPUNIT_ASSERT(result.hasError()); - CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O write") - != vespalib::string::npos); - - CPPUNIT_ASSERT(!env()._cache.contains(bucketId)); - - // Check that we still have first persisted put - memfile::MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(2345), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(*doc, *file->getDocument((*file)[0], ALL)); - } -} - -void -BasicOperationHandlerTest::testEraseFromCacheOnMaintainException() -{ - document::BucketId bucketId(16, 4); - - setupTestConfig(); - - getFakeClock()._absoluteTime = framework::MicroSecTime(2000 * 1000000); - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options) - .revertTimePeriod(framework::MicroSecTime(100000ULL * 1000000)) - .build()); - // Put a doc twice to allow for revert time compaction to be done - document::Document::SP doc1( - createRandomDocumentAtLocation(4, 2345, 1024, 1024)); - document::Document::SP doc2( - createRandomDocumentAtLocation(4, 2345, 1024, 1024)); - doPut(doc1, bucketId, Timestamp(1000 * 1000000)); - doPut(doc2, bucketId, Timestamp(1500 * 1000000)); - flush(bucketId); - env()._cache.clear(); - - options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options) - .revertTimePeriod(framework::MicroSecTime(100ULL * 1000000)) - .build()); - - env()._lazyFileFactory = - std::unique_ptr( - new SimulatedFailureLazyFile::Factory); - - spi::Result result = getPersistenceProvider().maintain(makeSpiBucket(bucketId), spi::HIGH); - CPPUNIT_ASSERT(result.hasError()); - CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O write") - != vespalib::string::npos); - - CPPUNIT_ASSERT(!env()._cache.contains(bucketId)); - - // Check that we still have both persisted puts - memfile::MemFilePtr file(getMemFile(bucketId)); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1000 * 1000000), (*file)[0].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(*doc1, *file->getDocument((*file)[0], ALL)); - CPPUNIT_ASSERT_EQUAL(Timestamp(1500 * 1000000), (*file)[1].getTimestamp()); - CPPUNIT_ASSERT_EQUAL(*doc2, *file->getDocument((*file)[1], ALL)); -} - -void -BasicOperationHandlerTest::testEraseFromCacheOnDeleteBucketException() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::Document::SP doc( - createRandomDocumentAtLocation(4, 2345, 1024, 1024)); - doPut(doc, bucketId, Timestamp(2345)); - flush(bucketId); - env()._cache.clear(); - - SimulatedFailureLazyFile::Factory* factory( - new SimulatedFailureLazyFile::Factory); - factory->setReadOpsBeforeFailure(0); - env()._lazyFileFactory = - std::unique_ptr(factory); - - // loadFile will fail - spi::Result result = getPersistenceProvider().deleteBucket(makeSpiBucket(bucketId), context); - CPPUNIT_ASSERT(result.hasError()); - CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O read") - != vespalib::string::npos); - - CPPUNIT_ASSERT(!env()._cache.contains(bucketId)); - -} - -void BasicOperationHandlerTest::list_buckets_returns_empty_set_for_non_default_bucketspace() { - document::BucketId bucket(16, 4); - doPut(createRandomDocumentAtLocation(4), bucket, Timestamp(4567), 0); - flush(bucket); - - auto buckets = getPersistenceProvider().listBuckets(document::FixedBucketSpaces::global_space(), spi::PartitionId(0)); - CPPUNIT_ASSERT_EQUAL(size_t(0), buckets.getList().size()); -} - -void BasicOperationHandlerTest::get_modified_buckets_returns_empty_set_for_non_default_bucketspace() { - env().addModifiedBucket(document::BucketId(16, 1234)); - auto buckets = getPersistenceProvider().getModifiedBuckets(document::FixedBucketSpaces::global_space()); - CPPUNIT_ASSERT_EQUAL(size_t(0), buckets.getList().size()); -} - -} - -} diff --git a/memfilepersistence/src/tests/spi/buffer_test.cpp b/memfilepersistence/src/tests/spi/buffer_test.cpp deleted file mode 100644 index bb31577bf17..00000000000 --- a/memfilepersistence/src/tests/spi/buffer_test.cpp +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include - -namespace storage { -namespace memfile { - -class BufferTest : public CppUnit::TestFixture -{ -public: - void getSizeReturnsInitiallyAllocatedSize(); - void getSizeReturnsUnAlignedSizeForMMappedAllocs(); - void resizeRetainsExistingDataWhenSizingUp(); - void resizeRetainsExistingDataWhenSizingDown(); - void bufferAddressIs512ByteAligned(); - - CPPUNIT_TEST_SUITE(BufferTest); - CPPUNIT_TEST(getSizeReturnsInitiallyAllocatedSize); - CPPUNIT_TEST(getSizeReturnsUnAlignedSizeForMMappedAllocs); - CPPUNIT_TEST(resizeRetainsExistingDataWhenSizingUp); - CPPUNIT_TEST(resizeRetainsExistingDataWhenSizingDown); - CPPUNIT_TEST(bufferAddressIs512ByteAligned); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(BufferTest); - -void -BufferTest::getSizeReturnsInitiallyAllocatedSize() -{ - Buffer buf(1234); - CPPUNIT_ASSERT_EQUAL(size_t(1234), buf.getSize()); -} - -void -BufferTest::getSizeReturnsUnAlignedSizeForMMappedAllocs() -{ - Buffer buf(vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE + 1); - CPPUNIT_ASSERT_EQUAL(size_t(vespalib::alloc::MemoryAllocator::HUGEPAGE_SIZE + 1), buf.getSize()); -} - -void -BufferTest::resizeRetainsExistingDataWhenSizingUp() -{ - std::string src = "hello world"; - Buffer buf(src.size()); - memcpy(buf.getBuffer(), src.data(), src.size()); - buf.resize(src.size() * 2); - CPPUNIT_ASSERT_EQUAL(src.size() * 2, buf.getSize()); - CPPUNIT_ASSERT_EQUAL(0, memcmp(buf.getBuffer(), src.data(), src.size())); -} - -void -BufferTest::resizeRetainsExistingDataWhenSizingDown() -{ - std::string src = "hello world"; - Buffer buf(src.size()); - memcpy(buf.getBuffer(), src.data(), src.size()); - buf.resize(src.size() / 2); - CPPUNIT_ASSERT_EQUAL(src.size() / 2, buf.getSize()); - CPPUNIT_ASSERT_EQUAL(0, memcmp(buf.getBuffer(), src.data(), src.size() / 2)); -} - -void -BufferTest::bufferAddressIs512ByteAligned() -{ - Buffer buf(32); - CPPUNIT_ASSERT(reinterpret_cast(buf.getBuffer()) % 512 == 0); -} - -} // memfile -} // storage - diff --git a/memfilepersistence/src/tests/spi/buffered_file_writer_test.cpp b/memfilepersistence/src/tests/spi/buffered_file_writer_test.cpp deleted file mode 100644 index 36270335fda..00000000000 --- a/memfilepersistence/src/tests/spi/buffered_file_writer_test.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include - -namespace storage { -namespace memfile { - -class BufferedFileWriterTest : public CppUnit::TestFixture -{ -public: - void noImplicitFlushingWhenDestructing(); - - CPPUNIT_TEST_SUITE(BufferedFileWriterTest); - CPPUNIT_TEST(noImplicitFlushingWhenDestructing); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(BufferedFileWriterTest); - -namespace { - -// Partial mock of vespalib::File. Unfortunately, there's currently no -// base interface to implement so have to override a class that already has -// implementation code present. -class MockFile : public vespalib::File -{ -public: - bool _didWrite; - - MockFile(const std::string& filename) - : File(filename), - _didWrite(false) - { - } - - void open(int flags, bool autoCreateDirectories) override { - (void) flags; - (void) autoCreateDirectories; - // Don't do anything here to prevent us from actually opening a file - // on disk. - } - - off_t write(const void *buf, size_t bufsize, off_t offset) override { - (void) buf; - (void) bufsize; - (void) offset; - _didWrite = true; - return 0; - } -}; - -} - -void -BufferedFileWriterTest::noImplicitFlushingWhenDestructing() -{ - MockFile file("foo"); - { - Buffer buffer(1024); - BufferedFileWriter writer(file, buffer, buffer.getSize()); - // Do a buffered write. This fits well within the buffer and should - // consequently not be immediately written out to the backing file. - writer.write("blarg", 5); - // Escape scope without having flushed anything. - } - // Since BufferedFileWriter is meant to be used with O_DIRECT files, - // flushing just implies writing rather than syncing (this is a half truth - // since you still sync directories etc to ensure metadata is written, but - // this constrained assumption works fine in the context of this test). - CPPUNIT_ASSERT(!file._didWrite); -} - -} // memfile -} // storage - diff --git a/memfilepersistence/src/tests/spi/iteratorhandlertest.cpp b/memfilepersistence/src/tests/spi/iteratorhandlertest.cpp deleted file mode 100644 index 418ead076fd..00000000000 --- a/memfilepersistence/src/tests/spi/iteratorhandlertest.cpp +++ /dev/null @@ -1,929 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include -#include -#include - -using storage::spi::test::makeSpiBucket; - -namespace storage { -namespace memfile { -namespace { - spi::LoadType defaultLoadType(0, "default"); -} - -class IteratorHandlerTest : public SingleDiskMemFileTestUtils -{ - CPPUNIT_TEST_SUITE(IteratorHandlerTest); - CPPUNIT_TEST(testCreateIterator); - CPPUNIT_TEST(testSomeSlotsRemovedBetweenInvocations); - CPPUNIT_TEST(testAllSlotsRemovedBetweenInvocations); - CPPUNIT_TEST(testIterateMetadataOnly); - CPPUNIT_TEST(testIterateHeadersOnly); - CPPUNIT_TEST(testIterateLargeDocument); - CPPUNIT_TEST(testDocumentsRemovedBetweenInvocations); - CPPUNIT_TEST(testUnrevertableRemoveBetweenInvocations); - CPPUNIT_TEST(testUnrevertableRemoveBetweenInvocationsIncludeRemoves); - CPPUNIT_TEST(testMatchTimestampRangeDocAltered); - CPPUNIT_TEST(testIterateAllVersions); - CPPUNIT_TEST(testFieldSetFiltering); - CPPUNIT_TEST(testIteratorInactiveOnException); - CPPUNIT_TEST(testDocsCachedBeforeDocumentSelection); - CPPUNIT_TEST(testTimestampRangeLimitedPrefetch); - CPPUNIT_TEST(testCachePrefetchRequirements); - CPPUNIT_TEST(testBucketEvictedFromCacheOnIterateException); - CPPUNIT_TEST_SUITE_END(); - -public: - void testCreateIterator(); - void testSomeSlotsRemovedBetweenInvocations(); - void testAllSlotsRemovedBetweenInvocations(); - void testIterateMetadataOnly(); - void testIterateHeadersOnly(); - void testIterateLargeDocument(); - void testDocumentsRemovedBetweenInvocations(); - void testUnrevertableRemoveBetweenInvocations(); - void testUnrevertableRemoveBetweenInvocationsIncludeRemoves(); - void testMatchTimestampRangeDocAltered(); - void testIterateAllVersions(); - void testFieldSetFiltering(); - void testIteratorInactiveOnException(); - void testDocsCachedBeforeDocumentSelection(); - void testTimestampRangeLimitedPrefetch(); - void testCachePrefetchRequirements(); - void testBucketEvictedFromCacheOnIterateException(); - - void setUp() override; - void tearDown() override; - - struct Chunk - { - std::vector _entries; - }; - -private: - spi::Selection createSelection(const std::string& docSel) const; - - - spi::CreateIteratorResult create( - const spi::Bucket& b, - const spi::Selection& sel, - spi::IncludedVersions versions = spi::NEWEST_DOCUMENT_ONLY, - const document::FieldSet& fieldSet = document::AllFields()) - { - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - return getPersistenceProvider().createIterator(b, fieldSet, sel, - versions, context); - } - - typedef std::pair DocAndTimestamp; - - std::vector feedDocs(size_t numDocs, - uint32_t minSize = 110, - uint32_t maxSize = 110); - - std::vector doIterate(spi::IteratorId id, - uint64_t maxByteSize, - size_t maxChunks = 0, - bool allowEmptyResult = false); - - void verifyDocs(const std::vector& wanted, - const std::vector& chunks, - const std::set& removes - = std::set()) const; - - void doTestUnrevertableRemoveBetweenInvocations(bool includeRemoves); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(IteratorHandlerTest); - -void -IteratorHandlerTest::setUp() -{ - SingleDiskMemFileTestUtils::setUp(); -} - -void -IteratorHandlerTest::tearDown() -{ - SingleDiskMemFileTestUtils::tearDown(); -} - -spi::Selection -IteratorHandlerTest::createSelection(const std::string& docSel) const -{ - return spi::Selection(spi::DocumentSelection(docSel)); -} - -void -IteratorHandlerTest::testCreateIterator() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 1234))); - - spi::CreateIteratorResult iter1(create(b, createSelection("true"))); - CPPUNIT_ASSERT_EQUAL(spi::IteratorId(1), iter1.getIteratorId()); - - spi::CreateIteratorResult iter2(create(b, createSelection("true"))); - CPPUNIT_ASSERT_EQUAL(spi::IteratorId(2), iter2.getIteratorId()); -} - -std::vector -IteratorHandlerTest::doIterate(spi::IteratorId id, - uint64_t maxByteSize, - size_t maxChunks, - bool allowEmptyResult) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - std::vector chunks; - - while (true) { - spi::IterateResult result(getPersistenceProvider().iterate( - id, maxByteSize, context)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT(result.getEntries().size() > 0 || allowEmptyResult); - - chunks.push_back(Chunk{result.steal_entries()}); - if (result.isCompleted() - || (maxChunks != 0 && chunks.size() >= maxChunks)) - { - break; - } - } - return chunks; -} - -namespace { - -size_t -getDocCount(const std::vector& chunks) -{ - size_t count = 0; - for (size_t i=0; i& entries) -{ - size_t ret = 0; - for (size_t i = 0; i < entries.size(); ++i) { - if (entries[i]->isRemove()) { - ++ret; - } - } - return ret; -} - -struct DocEntryIndirectTimestampComparator -{ - bool operator()(const spi::DocEntry::UP& e1, - const spi::DocEntry::UP& e2) const - { - return e1->getTimestamp() < e2->getTimestamp(); - } -}; - -std::vector -getEntriesFromChunks(const std::vector& chunks) -{ - std::vector ret; - for (size_t chunk = 0; chunk < chunks.size(); ++chunk) { - for (size_t i = 0; i < chunks[chunk]._entries.size(); ++i) { - ret.push_back(spi::DocEntry::UP(chunks[chunk]._entries[i]->clone())); - } - } - std::sort(ret.begin(), - ret.end(), - DocEntryIndirectTimestampComparator()); - return ret; -} - -const vespalib::LazyFile& -getFileHandle(const MemFile& mf1) -{ - return static_cast( - mf1.getMemFileIO()).getFileHandle(); -} - -const LoggingLazyFile& -getLoggerFile(const MemFile& file) -{ - return dynamic_cast(getFileHandle(file)); -} - -} - -void -IteratorHandlerTest::verifyDocs(const std::vector& wanted, - const std::vector& chunks, - const std::set& removes) const -{ - std::vector retrieved( - getEntriesFromChunks(chunks)); - size_t removeCount = getRemoveEntryCount(retrieved); - // Ensure that we've got the correct number of puts and removes - CPPUNIT_ASSERT_EQUAL(removes.size(), removeCount); - CPPUNIT_ASSERT_EQUAL(wanted.size(), retrieved.size() - removeCount); - - size_t wantedIdx = 0; - for (size_t i = 0; i < retrieved.size(); ++i) { - spi::DocEntry& entry(*retrieved[i]); - if (entry.getDocument() != 0) { - if (!(*wanted[wantedIdx].first == *entry.getDocument())) { - std::ostringstream ss; - ss << "Documents differ! Wanted:\n" - << wanted[wantedIdx].first->toString(true) - << "\n\nGot:\n" - << entry.getDocument()->toString(true); - CPPUNIT_FAIL(ss.str()); - } - CPPUNIT_ASSERT_EQUAL(wanted[wantedIdx].second, entry.getTimestamp()); - CPPUNIT_ASSERT_EQUAL(wanted[wantedIdx].first->serialize()->getLength() - + sizeof(spi::DocEntry), - size_t(entry.getSize())); - ++wantedIdx; - } else { - // Remove-entry - CPPUNIT_ASSERT(entry.getDocumentId() != 0); - CPPUNIT_ASSERT_EQUAL(entry.getDocumentId()->getSerializedSize() - + sizeof(spi::DocEntry), - size_t(entry.getSize())); - if (removes.find(entry.getDocumentId()->toString()) == removes.end()) { - std::ostringstream ss; - ss << "Got unexpected remove entry for document id " - << *entry.getDocumentId(); - CPPUNIT_FAIL(ss.str()); - } - } - } -} - -// Feed numDocs documents, starting from timestamp 1000 -std::vector -IteratorHandlerTest::feedDocs(size_t numDocs, - uint32_t minSize, - uint32_t maxSize) -{ - std::vector docs; - for (uint32_t i = 0; i < numDocs; ++i) { - docs.push_back( - DocAndTimestamp( - doPut(4, - framework::MicroSecTime(1000 + i), - minSize, - maxSize), - spi::Timestamp(1000 + i))); - } - flush(document::BucketId(16, 4)); - return docs; -} - -void -IteratorHandlerTest::testSomeSlotsRemovedBetweenInvocations() -{ - std::vector docs = feedDocs(100, 4096, 4096); - - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - spi::Selection sel(createSelection("true")); - - spi::CreateIteratorResult iter(create(b, sel)); - CPPUNIT_ASSERT(env()._cache.contains(b.getBucketId())); - - std::vector chunks = doIterate(iter.getIteratorId(), 10000, 25); - CPPUNIT_ASSERT_EQUAL(size_t(25), chunks.size()); - - { - MemFilePtr file(getMemFile(b.getBucketId())); - - for (int i = 0 ; i < 2; ++i) { - const MemSlot* slot = file->getSlotWithId(docs.front().first->getId()); - CPPUNIT_ASSERT(slot != 0); - file->removeSlot(*slot); - docs.erase(docs.begin()); - } - file->flushToDisk(); - } - - std::vector chunks2 = doIterate(iter.getIteratorId(), 10000); - CPPUNIT_ASSERT_EQUAL(size_t(24), chunks2.size()); - std::move(chunks2.begin(), chunks2.end(), std::back_inserter(chunks)); - - verifyDocs(docs, chunks); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - - // Bucket should not be evicted from cache during normal operation. - CPPUNIT_ASSERT(env()._cache.contains(b.getBucketId())); -} - -void -IteratorHandlerTest::testAllSlotsRemovedBetweenInvocations() -{ - std::vector docs = feedDocs(100, 4096, 4096); - - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - spi::Selection sel(createSelection("true")); - - spi::CreateIteratorResult iter(create(b, sel)); - - std::vector chunks = doIterate(iter.getIteratorId(), 1, 25); - CPPUNIT_ASSERT_EQUAL(size_t(25), chunks.size()); - - { - MemFilePtr file(getMemFile(b.getBucketId())); - - for (int i = 0 ; i < 75; ++i) { - const MemSlot* slot = file->getSlotWithId(docs[i].first->getId()); - CPPUNIT_ASSERT(slot != 0); - file->removeSlot(*slot); - } - file->flushToDisk(); - docs.erase(docs.begin(), docs.begin() + 75); - } - - std::vector chunks2 = doIterate(iter.getIteratorId(), 1, 0, true); - CPPUNIT_ASSERT_EQUAL(size_t(0), getDocCount(chunks2)); - verifyDocs(docs, chunks); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testIterateMetadataOnly() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - std::vector docs = feedDocs(10); - - CPPUNIT_ASSERT( - doUnrevertableRemove(b.getBucketId(), - docs[docs.size() - 2].first->getId(), - Timestamp(1008))); - - CPPUNIT_ASSERT( - doRemove(b.getBucketId(), - docs[docs.size() - 1].first->getId(), - framework::MicroSecTime(3001), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - - flush(b.getBucketId()); - - spi::Selection sel(createSelection("true")); - spi::CreateIteratorResult iter( - create(b, sel, spi::NEWEST_DOCUMENT_OR_REMOVE, document::NoFields())); - - std::vector chunks = doIterate(iter.getIteratorId(), 4096); - std::vector entries = getEntriesFromChunks(chunks); - CPPUNIT_ASSERT_EQUAL(docs.size(), entries.size()); - std::vector::const_iterator docIter( - docs.begin()); - for (size_t i = 0; i < entries.size(); ++i, ++docIter) { - const spi::DocEntry& entry = *entries[i]; - - CPPUNIT_ASSERT(entry.getDocument() == 0); - CPPUNIT_ASSERT(entry.getDocumentId() == 0); - if (i == 9) { - CPPUNIT_ASSERT(entry.isRemove()); - CPPUNIT_ASSERT_EQUAL(spi::Timestamp(3001), entry.getTimestamp()); - } else if (i == 8) { - CPPUNIT_ASSERT(entry.isRemove()); - CPPUNIT_ASSERT_EQUAL(spi::Timestamp(1008), entry.getTimestamp()); - } else { - CPPUNIT_ASSERT(!entry.isRemove()); - CPPUNIT_ASSERT_EQUAL(docIter->second, entry.getTimestamp()); - } - } - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testIterateHeadersOnly() -{ - std::vector docs = feedDocs(20); - // Remove all bodies. - for (size_t i = 0; i < docs.size(); ++i) { - clearBody(*docs[i].first); - } - - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - spi::Selection sel(createSelection("true")); - - spi::CreateIteratorResult iter(create(b, sel, spi::NEWEST_DOCUMENT_ONLY, - document::HeaderFields())); - - std::vector chunks = doIterate(iter.getIteratorId(), 1024); - verifyDocs(docs, chunks); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testIterateLargeDocument() -{ - std::vector docs = feedDocs(10, 10000, 10000); - std::vector largedoc; - largedoc.push_back(docs.back()); - - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - spi::Selection sel(createSelection("true")); - - spi::CreateIteratorResult iter(create(b, sel)); - - std::vector chunks = doIterate(iter.getIteratorId(), 100, 1); - verifyDocs(largedoc, chunks); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testDocumentsRemovedBetweenInvocations() -{ - int docCount = 100; - std::vector docs = feedDocs(docCount); - - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - spi::Selection sel(createSelection("true")); - - spi::CreateIteratorResult iter(create(b, sel)); - - std::vector chunks = doIterate(iter.getIteratorId(), 1, 25); - CPPUNIT_ASSERT_EQUAL(size_t(25), chunks.size()); - - // Remove a subset of the documents. We should still get all the - // original documents from the iterator, assuming no compactions. - std::vector removedDocs; - std::vector nonRemovedDocs; - for (int i = 0; i < docCount; ++i) { - if (i % 3 == 0) { - removedDocs.push_back(docs[i].first->getId()); - CPPUNIT_ASSERT(doRemove(b.getBucketId(), - removedDocs.back(), - framework::MicroSecTime(2000 + i), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - } else { - nonRemovedDocs.push_back(docs[i]); - } - } - flush(b.getBucketId()); - - std::vector chunks2 = doIterate(iter.getIteratorId(), 1); - CPPUNIT_ASSERT_EQUAL(size_t(75), chunks2.size()); - std::move(chunks2.begin(), chunks2.end(), std::back_inserter(chunks)); - - verifyDocs(docs, chunks); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::doTestUnrevertableRemoveBetweenInvocations(bool includeRemoves) -{ - int docCount = 100; - std::vector docs = feedDocs(docCount); - - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - spi::Selection sel(createSelection("true")); - spi::CreateIteratorResult iter( - create(b, sel, - includeRemoves ? - spi::NEWEST_DOCUMENT_OR_REMOVE : spi::NEWEST_DOCUMENT_ONLY)); - - std::vector chunks = doIterate(iter.getIteratorId(), 1, 25); - CPPUNIT_ASSERT_EQUAL(size_t(25), chunks.size()); - - // Remove a subset of the documents unrevertably. - std::vector removedDocs; - std::vector nonRemovedDocs; - for (int i = 0; i < docCount - 25; ++i) { - if (i < 10) { - removedDocs.push_back(docs[i].first->getId()); - CPPUNIT_ASSERT( - doUnrevertableRemove(b.getBucketId(), - removedDocs.back(), - Timestamp(1000+i))); - } else { - nonRemovedDocs.push_back(docs[i]); - } - } - flush(b.getBucketId()); - - std::vector chunks2 = doIterate(iter.getIteratorId(), 1); - std::vector entries = getEntriesFromChunks(chunks2); - if (!includeRemoves) { - CPPUNIT_ASSERT_EQUAL(nonRemovedDocs.size(), chunks2.size()); - verifyDocs(nonRemovedDocs, chunks2); - } else { - CPPUNIT_ASSERT_EQUAL(size_t(75), entries.size()); - for (int i = 0; i < docCount - 25; ++i) { - spi::DocEntry& entry(*entries[i]); - if (i < 10) { - CPPUNIT_ASSERT(entry.isRemove()); - } else { - CPPUNIT_ASSERT(!entry.isRemove()); - } - } - } - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testUnrevertableRemoveBetweenInvocations() -{ - doTestUnrevertableRemoveBetweenInvocations(false); -} - -void -IteratorHandlerTest::testUnrevertableRemoveBetweenInvocationsIncludeRemoves() -{ - doTestUnrevertableRemoveBetweenInvocations(true); -} - -void -IteratorHandlerTest::testMatchTimestampRangeDocAltered() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucketId(16, 4); - document::StringFieldValue updateValue1("update1"); - document::StringFieldValue updateValue2("update2"); - - Document::SP originalDoc = doPut(4, Timestamp(1234)); - - { - document::DocumentUpdate::SP update = createBodyUpdate( - originalDoc->getId(), updateValue1); - - spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(2345)); - CPPUNIT_ASSERT_EQUAL(1234, (int)result.getExistingTimestamp()); - } - - { - document::DocumentUpdate::SP update = createBodyUpdate( - originalDoc->getId(), updateValue2); - - spi::UpdateResult result = doUpdate(bucketId, update, Timestamp(3456)); - CPPUNIT_ASSERT_EQUAL(2345, (int)result.getExistingTimestamp()); - } - - CPPUNIT_ASSERT( - doRemove(bucketId, - originalDoc->getId(), - Timestamp(4567), - OperationHandler::PERSIST_REMOVE_IF_FOUND)); - flush(bucketId); - - spi::Bucket b(makeSpiBucket(bucketId)); - - { - spi::Selection sel(createSelection("true")); - sel.setFromTimestamp(spi::Timestamp(0)); - sel.setToTimestamp(spi::Timestamp(10)); - spi::CreateIteratorResult iter(create(b, sel)); - - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 4096, context)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT_EQUAL(size_t(0), result.getEntries().size()); - CPPUNIT_ASSERT(result.isCompleted()); - - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - } - - { - spi::Selection sel(createSelection("true")); - sel.setFromTimestamp(spi::Timestamp(10000)); - sel.setToTimestamp(spi::Timestamp(20000)); - spi::CreateIteratorResult iter(create(b, sel)); - - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 4096, context)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT_EQUAL(size_t(0), result.getEntries().size()); - CPPUNIT_ASSERT(result.isCompleted()); - - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - } - - { - spi::Selection sel(createSelection("true")); - sel.setFromTimestamp(spi::Timestamp(0)); - sel.setToTimestamp(spi::Timestamp(1234)); - spi::CreateIteratorResult iter(create(b, sel)); - - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 4096, context)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT_EQUAL(size_t(1), result.getEntries().size()); - CPPUNIT_ASSERT(result.isCompleted()); - - const Document& receivedDoc(*result.getEntries()[0]->getDocument()); - if (!(*originalDoc == receivedDoc)) { - std::ostringstream ss; - ss << "Documents differ! Wanted:\n" - << originalDoc->toString(true) - << "\n\nGot:\n" - << receivedDoc.toString(true); - CPPUNIT_FAIL(ss.str()); - } - - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - } - - { - spi::Selection sel(createSelection("true")); - sel.setFromTimestamp(spi::Timestamp(0)); - sel.setToTimestamp(spi::Timestamp(2345)); - spi::CreateIteratorResult iter(create(b, sel)); - - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 4096, context)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT_EQUAL(size_t(1), result.getEntries().size()); - CPPUNIT_ASSERT(result.isCompleted()); - - const Document& receivedDoc(*result.getEntries()[0]->getDocument()); - CPPUNIT_ASSERT(receivedDoc.getValue("content").get()); - CPPUNIT_ASSERT_EQUAL(updateValue1, - dynamic_cast( - *receivedDoc.getValue( - "content"))); - - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - } - - { - spi::Selection sel(createSelection("true")); - sel.setFromTimestamp(spi::Timestamp(0)); - sel.setToTimestamp(spi::Timestamp(3456)); - spi::CreateIteratorResult iter(create(b, sel)); - - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 4096, context)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT_EQUAL(size_t(1), result.getEntries().size()); - CPPUNIT_ASSERT(result.isCompleted()); - - const Document& receivedDoc(*result.getEntries()[0]->getDocument()); - CPPUNIT_ASSERT(receivedDoc.getValue("content").get()); - CPPUNIT_ASSERT_EQUAL(updateValue2, - dynamic_cast( - *receivedDoc.getValue( - "content"))); - - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - } -} - -void -IteratorHandlerTest::testIterateAllVersions() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - std::vector docs; - - Document::SP originalDoc(createRandomDocumentAtLocation( - 4, 1001, 110, 110)); - - doPut(originalDoc, framework::MicroSecTime(1001), 0); - - document::StringFieldValue updateValue1("update1"); - { - document::DocumentUpdate::SP update = createBodyUpdate( - originalDoc->getId(), updateValue1); - - spi::UpdateResult result = doUpdate(b.getBucketId(), update, Timestamp(2345)); - CPPUNIT_ASSERT_EQUAL(1001, (int)result.getExistingTimestamp()); - } - flush(b.getBucketId()); - - Document::SP updatedDoc(new Document(*originalDoc)); - updatedDoc->setValue("content", document::StringFieldValue("update1")); - docs.push_back(DocAndTimestamp(originalDoc, spi::Timestamp(1001))); - docs.push_back(DocAndTimestamp(updatedDoc, spi::Timestamp(2345))); - - spi::Selection sel(createSelection("true")); - spi::CreateIteratorResult iter(create(b, sel, spi::ALL_VERSIONS)); - - std::vector chunks = doIterate(iter.getIteratorId(), 4096); - verifyDocs(docs, chunks); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testFieldSetFiltering() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - Document::SP doc(createRandomDocumentAtLocation( - 4, 1001, 110, 110)); - doc->setValue(doc->getField("headerval"), document::IntFieldValue(42)); - doc->setValue(doc->getField("hstringval"), - document::StringFieldValue("groovy, baby!")); - doc->setValue(doc->getField("content"), - document::StringFieldValue("fancy content")); - doPut(doc, framework::MicroSecTime(1001), 0); - flush(b.getBucketId()); - - document::FieldSetRepo repo; - spi::Selection sel(createSelection("true")); - spi::CreateIteratorResult iter( - create(b, sel, spi::NEWEST_DOCUMENT_ONLY, - *repo.parse(*getTypeRepo(), "testdoctype1:hstringval,content"))); - std::vector entries( - getEntriesFromChunks(doIterate(iter.getIteratorId(), 4096))); - CPPUNIT_ASSERT_EQUAL(size_t(1), entries.size()); - CPPUNIT_ASSERT_EQUAL(std::string("content: fancy content\n" - "hstringval: groovy, baby!\n"), - stringifyFields(*entries[0]->getDocument())); -} - -void -IteratorHandlerTest::testIteratorInactiveOnException() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - feedDocs(10); - - env()._cache.clear(); - - simulateIoErrorsForSubsequentlyOpenedFiles(IoErrors().afterReads(1)); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - spi::CreateIteratorResult iter(create(b, createSelection("true"))); - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 100000, context)); - CPPUNIT_ASSERT(result.hasError()); - // Check that iterator is marked as inactive - const SharedIteratorHandlerState& state( - getPersistenceProvider().getIteratorHandler().getState()); - CPPUNIT_ASSERT(state._iterators.find(iter.getIteratorId().getValue()) - != state._iterators.end()); - CPPUNIT_ASSERT(state._iterators.find(iter.getIteratorId().getValue()) - ->second.isActive() == false); - - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); -} - -void -IteratorHandlerTest::testDocsCachedBeforeDocumentSelection() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - std::vector docs = feedDocs(100, 4096, 4096); - - env()._cache.clear(); - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options).maximumReadThroughGap(1024*1024).build()); - env()._lazyFileFactory = std::unique_ptr( - new LoggingLazyFile::Factory()); - - spi::Selection sel(createSelection("id.user=4")); - spi::CreateIteratorResult iter(create(b, sel, spi::NEWEST_DOCUMENT_ONLY, - document::BodyFields())); - - std::vector chunks = doIterate(iter.getIteratorId(), 4096); - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - { - MemFilePtr file(getMemFile(b.getBucketId())); - // Should have 3 read ops; metadata, (precached) headers and bodies - CPPUNIT_ASSERT_EQUAL(size_t(3), - getLoggerFile(*file).operations.size()); - } -} - -void -IteratorHandlerTest::testTimestampRangeLimitedPrefetch() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - // Feed docs with timestamp range [1000, 1100) - feedDocs(100, 4096, 4096); - - env()._cache.clear(); - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options).maximumReadThroughGap(512).build()); - env()._lazyFileFactory = std::unique_ptr( - new LoggingLazyFile::Factory()); - - spi::Selection sel(createSelection("id.user=4")); - sel.setFromTimestamp(spi::Timestamp(1050)); - sel.setToTimestamp(spi::Timestamp(1059)); - spi::CreateIteratorResult iter(create(b, sel, spi::NEWEST_DOCUMENT_ONLY, - document::BodyFields())); - std::vector chunks = doIterate(iter.getIteratorId(), 4096); - CPPUNIT_ASSERT_EQUAL(size_t(10), getDocCount(chunks)); - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().destroyIterator(iter.getIteratorId(), context); - // Iterate over all slots, ensuring that only those that fall within the - // timestamp range have actually been cached. - { - MemFilePtr file(getMemFile(b.getBucketId())); - // Should have 3 read ops; metadata, (precached) headers and bodies - CPPUNIT_ASSERT_EQUAL(size_t(3), - getLoggerFile(*file).operations.size()); - for (size_t i = 0; i < file->getSlotCount(); ++i) { - const MemSlot& slot((*file)[i]); - if (slot.getTimestamp() >= Timestamp(1050) - && slot.getTimestamp() <= Timestamp(1059)) - { - CPPUNIT_ASSERT(file->partAvailable(slot, HEADER)); - CPPUNIT_ASSERT(file->partAvailable(slot, BODY)); - } else { - CPPUNIT_ASSERT(!file->partAvailable(slot, HEADER)); - CPPUNIT_ASSERT(!file->partAvailable(slot, BODY)); - } - } - } -} - -void -IteratorHandlerTest::testCachePrefetchRequirements() -{ - document::select::Parser parser( - env().repo(), env()._bucketFactory); - { - // No prefetch required. - // NOTE: since stuff like id.user=1234 won't work, we have to handle - // that explicitly in createIterator based on the assumption that a - // non-empty document selection at _least_ requires header to be read. - std::unique_ptr sel( - parser.parse("true")); - CachePrefetchRequirements req( - CachePrefetchRequirements::createFromSelection(env().repo(), - *sel)); - CPPUNIT_ASSERT(!req.isHeaderPrefetchRequired()); - CPPUNIT_ASSERT(!req.isBodyPrefetchRequired()); - } - - { - // Header prefetch required. - std::unique_ptr sel( - parser.parse("testdoctype1.hstringval='blarg'")); - CachePrefetchRequirements req( - CachePrefetchRequirements::createFromSelection(env().repo(), - *sel)); - CPPUNIT_ASSERT(req.isHeaderPrefetchRequired()); - CPPUNIT_ASSERT(!req.isBodyPrefetchRequired()); - } - - { - // Body prefetch required. - std::unique_ptr sel( - parser.parse("testdoctype1.content='foobar'")); - CachePrefetchRequirements req( - CachePrefetchRequirements::createFromSelection(env().repo(), - *sel)); - CPPUNIT_ASSERT(!req.isHeaderPrefetchRequired()); - CPPUNIT_ASSERT(req.isBodyPrefetchRequired()); - } -} - -void -IteratorHandlerTest::testBucketEvictedFromCacheOnIterateException() -{ - spi::Bucket b(makeSpiBucket(BucketId(16, 4))); - feedDocs(10); - env()._cache.clear(); - - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - spi::CreateIteratorResult iter(create(b, createSelection("true"))); - simulateIoErrorsForSubsequentlyOpenedFiles(IoErrors().afterReads(1)); - spi::IterateResult result(getPersistenceProvider().iterate( - iter.getIteratorId(), 100000, context)); - CPPUNIT_ASSERT(result.hasError()); - - // This test is actually a bit disingenuous since calling iterate will - // implicitly invoke maintain() on an IO exception, which will subsequently - // evict the bucket due to the exception happening again in its context. - CPPUNIT_ASSERT(!env()._cache.contains(b.getBucketId())); -} - -} -} diff --git a/memfilepersistence/src/tests/spi/joinoperationhandlertest.cpp b/memfilepersistence/src/tests/spi/joinoperationhandlertest.cpp deleted file mode 100644 index 2a9395767af..00000000000 --- a/memfilepersistence/src/tests/spi/joinoperationhandlertest.cpp +++ /dev/null @@ -1,504 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "memfiletestutils.h" -#include -#include -#include - -using document::DocumentType; -using storage::spi::test::makeSpiBucket; - -namespace storage { -namespace memfile { -namespace { - spi::LoadType defaultLoadType(0, "default"); -} - -class JoinOperationHandlerTest : public MemFileTestUtils -{ - CPPUNIT_TEST_SUITE(JoinOperationHandlerTest); - CPPUNIT_TEST(testSimple); - CPPUNIT_TEST(testTargetExists); - CPPUNIT_TEST(testTargetWithOverlap); - CPPUNIT_TEST(testMultiDisk); - CPPUNIT_TEST(testMultiDiskFlushed); - CPPUNIT_TEST(testInternalJoin); - CPPUNIT_TEST(testInternalJoinDiskFull); - CPPUNIT_TEST(testTargetIoWriteExceptionEvictsTargetFromCache); - CPPUNIT_TEST(test1stSourceIoReadExceptionEvictsSourceFromCache); - CPPUNIT_TEST(test2ndSourceExceptionEvictsExistingTargetFromCache); - CPPUNIT_TEST_SUITE_END(); - -public: - void testSimple(); - void testTargetExists(); - void testTargetWithOverlap(); - void testMultiDisk(); - void testMultiDiskFlushed(); - void testInternalJoin(); - void testInternalJoinDiskFull(); - void testTargetIoWriteExceptionEvictsTargetFromCache(); - void test1stSourceIoReadExceptionEvictsSourceFromCache(); - void test2ndSourceExceptionEvictsExistingTargetFromCache(); - - void insertDocumentInBucket(uint64_t location, - Timestamp timestamp, - document::BucketId bucket); - -private: - void feedSingleDisk(); - void feedMultiDisk(); - std::string getStandardMemFileStatus(uint32_t disk = 0); - - spi::Result doJoin(const document::BucketId to, - const document::BucketId from1, - const document::BucketId from2); -}; - -namespace { - -document::BucketId TARGET = document::BucketId(15, 4); -document::BucketId SOURCE1 = document::BucketId(16, 4); -document::BucketId SOURCE2 = document::BucketId(16, (uint64_t)4 | ((uint64_t)1 << 15)); -} - -CPPUNIT_TEST_SUITE_REGISTRATION(JoinOperationHandlerTest); - -void -JoinOperationHandlerTest::feedSingleDisk() -{ - for (uint32_t i = 0; i < 100; i++) { - std::ostringstream ost; - ost << "userdoc:storage_test:1234:" << i; - const DocumentType& type( - *getTypeRepo()->getDocumentType("testdoctype1")); - document::Document::SP doc( - new document::Document(type, document::DocumentId(ost.str()))); - - document::BucketId bucket( - getBucketIdFactory().getBucketId(doc->getId())); - bucket.setUsedBits(33); - doPut(doc, Timestamp(1000 + i), 0, 33); - flush(bucket); - } -} - -void -JoinOperationHandlerTest::feedMultiDisk() -{ - for (uint32_t i = 0; i < 100; i += 2) { - doPutOnDisk(7, 4 | (1 << 15), Timestamp(1000 + i)); - } - flush(SOURCE2); - - for (uint32_t i = 1; i < 100; i += 2) { - doPutOnDisk(4, 4, Timestamp(1000 + i)); - } - flush(SOURCE1); - - { - MemFilePtr file(getMemFile(SOURCE1, 4)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(4, (int)file->getDisk()); - } - - { - MemFilePtr file(getMemFile(SOURCE2, 7)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(7, (int)file->getDisk()); - } -} - -std::string -JoinOperationHandlerTest::getStandardMemFileStatus(uint32_t disk) -{ - std::ostringstream ost; - - ost << getMemFileStatus(TARGET, disk) << "\n" - << getMemFileStatus(SOURCE1, disk ) << "\n" - << getMemFileStatus(SOURCE2, disk) << "\n"; - - return ost.str(); -} - -void -JoinOperationHandlerTest::insertDocumentInBucket( - uint64_t location, - Timestamp timestamp, - document::BucketId bucket) -{ - Document::SP doc( - createRandomDocumentAtLocation( - location, timestamp.getTime(), 100, 100)); - doPut(doc, bucket, timestamp); -} - -spi::Result -JoinOperationHandlerTest::doJoin(const document::BucketId to, - const document::BucketId from1, - const document::BucketId from2) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - return getPersistenceProvider().join( - makeSpiBucket(from1), - makeSpiBucket(from2), - makeSpiBucket(to), - context); -} - -void -JoinOperationHandlerTest::testSimple() -{ - setupDisks(1); - feedSingleDisk(); - - { - MemFilePtr file(getMemFile(document::BucketId(33, 1234))); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - } - - { - MemFilePtr file(getMemFile(document::BucketId(33, (uint64_t)1234 | ((uint64_t)1 << 32)))); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - } - - spi::Result result = - doJoin(document::BucketId(32, 1234), - document::BucketId(33, 1234), - document::BucketId(33, (uint64_t)1234 | ((uint64_t)1 << 32))); - - { - MemFilePtr file(getMemFile(document::BucketId(32, (uint64_t)1234))); - CPPUNIT_ASSERT_EQUAL(100, (int)file->getSlotCount()); - CPPUNIT_ASSERT(!file->slotsAltered()); - } -} - -void -JoinOperationHandlerTest::testTargetExists() -{ - setupDisks(1); - - for (uint32_t i = 0; i < 100; i += 2) { - doPut(4 | (1 << 15), Timestamp(1000 + i)); - } - flush(SOURCE2); - - for (uint32_t i = 1; i < 100; i += 2) { - doPut(4, Timestamp(1000 + i)); - } - flush(SOURCE1); - - for (uint32_t i = 0; i < 100; i++) { - uint32_t location = 4; - if (i % 2 == 0) { - location |= (1 << 15); - } - - insertDocumentInBucket(location, Timestamp(500 + i), TARGET); - } - flush(TARGET); - - doJoin(TARGET, SOURCE1, SOURCE2); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x3c00000000000004): 200,0\n" - "BucketId(0x4000000000000004): 0,0\n" - "BucketId(0x4000000000008004): 0,0\n"), - getStandardMemFileStatus()); -} - -void -JoinOperationHandlerTest::testTargetWithOverlap() -{ - setupDisks(1); - - for (uint32_t i = 0; i < 100; i += 2) { - doPut(4 | (1 << 15), Timestamp(1000 + i)); - } - flush(SOURCE2); - - for (uint32_t i = 1; i < 100; i += 2) { - doPut(4, Timestamp(1000 + i)); - } - flush(SOURCE1); - - for (uint32_t i = 0; i < 100; i++) { - uint32_t location = 4; - if (i % 2 == 0) { - location |= (1 << 15); - } - - insertDocumentInBucket(location, Timestamp(950 + i), TARGET); - } - flush(TARGET); - - doJoin(TARGET, SOURCE1, SOURCE2); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x3c00000000000004): 150,0\n" - "BucketId(0x4000000000000004): 0,0\n" - "BucketId(0x4000000000008004): 0,0\n"), - getStandardMemFileStatus()); -} - -void -JoinOperationHandlerTest::testMultiDisk() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(10); - feedMultiDisk(); - - getPersistenceProvider().join(makeSpiBucket(SOURCE2, spi::PartitionId(7)), - makeSpiBucket(SOURCE1, spi::PartitionId(4)), - makeSpiBucket(TARGET, spi::PartitionId(3)), - context); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x3c00000000000004): 100,3\n" - "BucketId(0x4000000000000004): 0,0\n" - "BucketId(0x4000000000008004): 0,0\n"), - getStandardMemFileStatus()); -} - -void -JoinOperationHandlerTest::testMultiDiskFlushed() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(10); - feedMultiDisk(); - - // Flush everything to disk, to check that we can join even - // if it's not in cache before. - env()._cache.flushDirtyEntries(); - env()._cache.clear(); - - getPersistenceProvider().join(makeSpiBucket(SOURCE2, spi::PartitionId(7)), - makeSpiBucket(SOURCE1, spi::PartitionId(4)), - makeSpiBucket(TARGET, spi::PartitionId(3)), - context); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x3c00000000000004): 100,3\n" - "BucketId(0x4000000000000004): 0,3\n" - "BucketId(0x4000000000008004): 0,3\n"), - getStandardMemFileStatus(3)); -} - -void -JoinOperationHandlerTest::testInternalJoin() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(10); - - for (uint32_t i = 4; i < 6; i++) { - for (uint32_t j = 0; j < 10; j++) { - uint32_t location = 4; - doPutOnDisk(i, location, Timestamp(i * 1000 + j)); - } - flush(document::BucketId(16, 4), i); - env()._cache.clear(); - } - - std::string fileName1 = - env().calculatePathInDir(SOURCE1, (*env()._mountPoints)[4]); - std::string fileName2 = - env().calculatePathInDir(SOURCE1, (*env()._mountPoints)[5]); - - CPPUNIT_ASSERT(vespalib::stat(fileName1).get()); - vespalib::FileInfo::UP file2(vespalib::stat(fileName2)); - - CPPUNIT_ASSERT(file2.get()); - CPPUNIT_ASSERT(file2->_size > 0); - - PartitionMonitor* mon = env().getDirectory(5).getPartition().getMonitor(); - // Set disk under 80% full. Over 80%, we shouldn't move buckets to the target. - mon->setStatOncePolicy(); - mon->overrideRealStat(512, 100000, 50000); - CPPUNIT_ASSERT(!mon->isFull(0, .80f)); - - getPersistenceProvider().join(makeSpiBucket(SOURCE1, spi::PartitionId(4)), - makeSpiBucket(SOURCE1, spi::PartitionId(4)), - makeSpiBucket(SOURCE1, spi::PartitionId(5)), - context); - - env()._cache.clear(); - - CPPUNIT_ASSERT(!vespalib::stat(fileName1).get()); - CPPUNIT_ASSERT(vespalib::stat(fileName2).get()); -} - -void -JoinOperationHandlerTest::testInternalJoinDiskFull() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(10); - - for (uint32_t i = 4; i < 6; i++) { - for (uint32_t j = 0; j < 10; j++) { - uint32_t location = 4; - doPutOnDisk(i, location, Timestamp(i * 1000 + j)); - } - flush(document::BucketId(16, 4), i); - env()._cache.clear(); - } - - std::string fileName1 = - env().calculatePathInDir(SOURCE1, (*env()._mountPoints)[4]); - std::string fileName2 = - env().calculatePathInDir(SOURCE1, (*env()._mountPoints)[5]); - - CPPUNIT_ASSERT(vespalib::stat(fileName1).get()); - vespalib::FileInfo::UP file2(vespalib::stat(fileName2)); - - CPPUNIT_ASSERT(file2.get()); - CPPUNIT_ASSERT(file2->_size > 0); - - PartitionMonitor* mon = env().getDirectory(5).getPartition().getMonitor(); - // Set disk to 81% full. Over 80%, we shouldn't move buckets to the target. - mon->setStatOncePolicy(); - mon->overrideRealStat(512, 100000, 81000); - CPPUNIT_ASSERT(!mon->isFull()); - CPPUNIT_ASSERT(mon->isFull(0, .08f)); - - spi::Result result = - getPersistenceProvider().join(makeSpiBucket(SOURCE1, spi::PartitionId(4)), - makeSpiBucket(SOURCE1, spi::PartitionId(4)), - makeSpiBucket(SOURCE1, spi::PartitionId(5)), - context); - - CPPUNIT_ASSERT(result.hasError()); -} - -void -JoinOperationHandlerTest::testTargetIoWriteExceptionEvictsTargetFromCache() -{ - setupDisks(1); - feedSingleDisk(); - - document::BucketId src1(33, 1234); - document::BucketId src2(33, 1234ULL | (1ULL << 32)); - document::BucketId target(32, 1234); - - CPPUNIT_ASSERT(env()._cache.contains(src1)); - CPPUNIT_ASSERT(env()._cache.contains(src2)); - CPPUNIT_ASSERT(!env()._cache.contains(target)); - - // Reading existing (fully cached) files will go fine, but writing - // new file will not. - simulateIoErrorsForSubsequentlyOpenedFiles(); - - spi::Result result = doJoin(target, src1, src2); - CPPUNIT_ASSERT(result.hasError()); - CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O write") - != vespalib::string::npos); - - CPPUNIT_ASSERT(!env()._cache.contains(target)); - // NOTE: since we end up renaming src1 -> target during the first - // iteration of join, src1 will actually be empty. This should not - // matter since the service layer will query the bucket info for - // all these afterwards and will thus pick up on this automatically. - unSimulateIoErrorsForSubsequentlyOpenedFiles(); - { - MemFilePtr file(getMemFile(src1)); - CPPUNIT_ASSERT_EQUAL(0, (int)file->getSlotCount()); - CPPUNIT_ASSERT(!file->slotsAltered()); - } - { - MemFilePtr file(getMemFile(src2)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - CPPUNIT_ASSERT(!file->slotsAltered()); - } - { - MemFilePtr file(getMemFile(target)); - // Renamed from src1 - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - CPPUNIT_ASSERT(!file->slotsAltered()); - } -} - -void -JoinOperationHandlerTest::test1stSourceIoReadExceptionEvictsSourceFromCache() -{ - setupDisks(1); - feedSingleDisk(); - - document::BucketId src1(33, 1234); - document::BucketId src2(33, 1234ULL | (1ULL << 32)); - document::BucketId target(32, 1234); - - env()._cache.clear(); - // Allow for reading in initial metadata so that loadFile itself doesn't - // fail. This could otherwise cause a false negative since that happens - // during initial cache lookup on a cache miss, at which point any - // exception will always stop a file from being added to the cache. Here - // we want to test the case where a file has been successfully hoisted - // out of the cache initially. - simulateIoErrorsForSubsequentlyOpenedFiles(IoErrors().afterReads(1)); - - spi::Result result = doJoin(target, src1, src2); - CPPUNIT_ASSERT(result.hasError()); - CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O read") - != vespalib::string::npos); - - CPPUNIT_ASSERT(!env()._cache.contains(src1)); - CPPUNIT_ASSERT(!env()._cache.contains(src2)); - CPPUNIT_ASSERT(!env()._cache.contains(target)); -} - -/** - * It must be exception safe for any source bucket to throw an exception during - * processing. Otherwise the node will core due to cache sanity checks. - * - * See VESPA-674 for context. In this scenario, it was not possible to write - * to the target file when attempting to join in the 2nd source bucket due to - * the disk fill ratio exceeding configured limits. - */ -void -JoinOperationHandlerTest::test2ndSourceExceptionEvictsExistingTargetFromCache() -{ - setupDisks(1); - feedSingleDisk(); - - constexpr uint64_t location = 1234; - - document::BucketId src1(33, location); - document::BucketId src2(33, location | (1ULL << 32)); - document::BucketId target(32, location); - - // Ensure target file is _not_ empty so that copySlots is triggered for - // each source bucket (rather than just renaming the file, which does not - // invoke the file read/write paths). - insertDocumentInBucket(location, Timestamp(100000), target); - flush(target); - - env()._cache.clear(); - // File rewrites are buffered before ever reaching the failure simulation - // layer, so only 1 actual write is used to flush the target file after - // the first source file has been processed. Attempting to flush the writes - // for the second source file should fail with an exception. - simulateIoErrorsForSubsequentlyOpenedFiles( - IoErrors().afterReads(INT_MAX).afterWrites(1)); - - spi::Result result = doJoin(target, src1, src2); - CPPUNIT_ASSERT(result.hasError()); - CPPUNIT_ASSERT(result.getErrorMessage().find("A simulated I/O write") - != vespalib::string::npos); - - CPPUNIT_ASSERT(!env()._cache.contains(src1)); - CPPUNIT_ASSERT(!env()._cache.contains(src2)); - CPPUNIT_ASSERT(!env()._cache.contains(target)); -} - -} - -} diff --git a/memfilepersistence/src/tests/spi/logginglazyfile.h b/memfilepersistence/src/tests/spi/logginglazyfile.h deleted file mode 100644 index d525f85b165..00000000000 --- a/memfilepersistence/src/tests/spi/logginglazyfile.h +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include -#include - -namespace storage::memfile { - -class LoggingLazyFile : public vespalib::LazyFile { -public: - class Factory : public Environment::LazyFileFactory { - public: - vespalib::LazyFile::UP createFile(const std::string& fileName) const override { - return vespalib::LazyFile::UP( - new LoggingLazyFile(fileName, vespalib::File::DIRECTIO)); - } - }; - - enum OpType { - READ = 0, - WRITE - }; - - struct Entry { - OpType opType; - size_t bufsize; - off_t offset; - - std::string toString() const { - std::ostringstream ost; - ost << (opType == READ ? "Reading " : "Writing ") - << bufsize - << " bytes at " - << offset; - return ost.str(); - } - }; - - mutable std::vector operations; - - LoggingLazyFile(const std::string& filename, int flags) - : LazyFile(filename, flags) {}; - - size_t getOperationCount() const { - return operations.size(); - } - - off_t write(const void *buf, size_t bufsize, off_t offset) override { - Entry e; - e.opType = WRITE; - e.bufsize = bufsize; - e.offset = offset; - - operations.push_back(e); - - return vespalib::LazyFile::write(buf, bufsize, offset); - } - - size_t read(void *buf, size_t bufsize, off_t offset) const override { - Entry e; - e.opType = READ; - e.bufsize = bufsize; - e.offset = offset; - - operations.push_back(e); - - return vespalib::LazyFile::read(buf, bufsize, offset); - } - - std::string toString() const { - std::ostringstream ost; - for (uint32_t i = 0; i < operations.size(); i++) { - ost << operations[i].toString() << "\n"; - } - - return ost.str(); - } - -}; - -} diff --git a/memfilepersistence/src/tests/spi/memcachetest.cpp b/memfilepersistence/src/tests/spi/memcachetest.cpp deleted file mode 100644 index 5e9f1a28225..00000000000 --- a/memfilepersistence/src/tests/spi/memcachetest.cpp +++ /dev/null @@ -1,397 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include - - -namespace storage { -namespace memfile { - -class MemCacheTest : public SingleDiskMemFileTestUtils -{ - CPPUNIT_TEST_SUITE(MemCacheTest); - CPPUNIT_TEST(testSimpleLRU); - CPPUNIT_TEST(testCacheSize); - CPPUNIT_TEST(testEvictBody); - CPPUNIT_TEST(testEvictHeader); - CPPUNIT_TEST(testKeepBodyWhenLessThanOneFourth); - CPPUNIT_TEST(testComplexEviction); - CPPUNIT_TEST(testEraseEmptyOnReturn); - CPPUNIT_TEST(testDeleteDoesNotReAddMemoryUsage); - CPPUNIT_TEST(testEraseDoesNotReAddMemoryUsage); - CPPUNIT_TEST(testGetWithNoCreation); - CPPUNIT_TEST_SUITE_END(); - -public: - void testSimpleLRU(); - void testCacheSize(); - void testReduceCacheSizeCallback(); - void testReduceCacheSizeCallbackWhileActive(); - void testEvictBody(); - void testEvictHeader(); - void testKeepBodyWhenLessThanOneFourth(); - void testComplexEviction(); - void testEraseEmptyOnReturn(); - void testDeleteDoesNotReAddMemoryUsage(); - void testEraseDoesNotReAddMemoryUsage(); - void testGetWithNoCreation(); - -private: - framework::defaultimplementation::ComponentRegisterImpl::UP _register; - framework::Component::UP _component; - FakeClock::UP _clock; - std::unique_ptr _metrics; - - std::unique_ptr _cache; - - void setSize(const document::BucketId& id, - uint64_t metaSize, - uint64_t headerSz = 0, - uint64_t bodySz = 0, - bool createIfNotInCache = true) - { - MemFilePtr file(_cache->get(id, env(), env().getDirectory(), - createIfNotInCache)); - CPPUNIT_ASSERT(file.get()); - - file->_cacheSizeOverride.metaSize = metaSize; - file->_cacheSizeOverride.headerSize = headerSz; - file->_cacheSizeOverride.bodySize = bodySz; - } - - std::string - getBucketStatus(uint32_t buckets) - { - std::ostringstream ost; - for (uint32_t i = 1; i < buckets + 1; i++) { - document::BucketId id(16, i); - ost << id << " "; - if (!_cache->contains(id)) { - ost << "\n"; - } else { - MemFilePtr file(_cache->get(id, env(), env().getDirectory())); - if (file->_cacheSizeOverride.bodySize > 0) { - ost << "body,"; - } - if (file->_cacheSizeOverride.headerSize > 0) { - ost << "header\n"; - } else { - ost << "meta only\n"; - } - } - } - - return ost.str(); - } - - uint64_t cacheSize() { - return _cache->size(); - } - - document::BucketId getLRU() { - return _cache->getLeastRecentlyUsedBucket()->_bid; - } - - void setCacheSize(uint64_t sz) { - MemFileCache::MemoryUsage usage; - usage.metaSize = sz / 3; - usage.headerSize = sz / 3; - usage.bodySize = sz - usage.metaSize - usage.headerSize; - - _cache->setCacheSize(usage); - } - - void stealMemory(uint64_t memToSteal) { - setCacheSize(_cache->getCacheSize() - memToSteal); - } - - void setup(uint64_t maxMemory) { - tearDown(); - _register.reset( - new framework::defaultimplementation::ComponentRegisterImpl); - _clock.reset(new FakeClock); - _register->setClock(*_clock); - _component.reset(new framework::Component(*_register, "testcomponent")); - _metrics.reset(new MemFilePersistenceMetrics(*_component)); - _cache.reset(new MemFileCache(*_register, _metrics->_cache)); - setCacheSize(maxMemory); - } - -public: - void tearDown() override { - _cache.reset(0); - _metrics.reset(0); - _component.reset(0); - _register.reset(0); - _clock.reset(0); - } -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemCacheTest); - -namespace { - FakeClock clock; -} - -void -MemCacheTest::testSimpleLRU() -{ - setup(2000); - - for (uint32_t i = 1; i < 4; i++) { - setSize(document::BucketId(16, i), 100); - } - - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 1), getLRU()); - - setSize(document::BucketId(16, 1), 100); - - CPPUNIT_ASSERT_EQUAL(1UL, _cache->getMetrics().hits.getValue()); - CPPUNIT_ASSERT_EQUAL(document::BucketId(16, 2), getLRU()); -} - -void -MemCacheTest::testCacheSize() -{ - setup(400); - - setSize(document::BucketId(16, 2), 100); - setSize(document::BucketId(16, 1), 150); - - CPPUNIT_ASSERT_EQUAL(0UL, _cache->getMetrics().hits.getValue()); - CPPUNIT_ASSERT_EQUAL(2UL, _cache->getMetrics().misses.getValue()); - - CPPUNIT_ASSERT_EQUAL(250ul, cacheSize()); - - setSize(document::BucketId(16, 1), 200); - - CPPUNIT_ASSERT_EQUAL(1UL, _cache->getMetrics().hits.getValue()); - CPPUNIT_ASSERT_EQUAL(2UL, _cache->getMetrics().misses.getValue()); - - CPPUNIT_ASSERT_EQUAL(300ul, cacheSize()); - - CPPUNIT_ASSERT(_cache->contains(document::BucketId(16, 2))); - CPPUNIT_ASSERT(_cache->contains(document::BucketId(16, 1))); - - setSize(document::BucketId(16, 1), 301); - - CPPUNIT_ASSERT_EQUAL(2UL, _cache->getMetrics().hits.getValue()); - CPPUNIT_ASSERT_EQUAL(2UL, _cache->getMetrics().misses.getValue()); - - CPPUNIT_ASSERT(!_cache->contains(document::BucketId(16, 2))); - CPPUNIT_ASSERT(_cache->contains(document::BucketId(16, 1))); - - _cache->clear(); - CPPUNIT_ASSERT_EQUAL(0ul, cacheSize()); -} - -void -MemCacheTest::testEvictBody() -{ - setup(1400); - - CPPUNIT_ASSERT_EQUAL(0UL, _cache->getMetrics().body_evictions.getValue()); - - setSize(BucketId(16, 1), 150, 100, 0); - setSize(BucketId(16, 2), 100, 100, 900); - - CPPUNIT_ASSERT_EQUAL(1350ul, cacheSize()); - - stealMemory(150); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) header\n" - "BucketId(0x4000000000000002) header\n"), - getBucketStatus(2)); - CPPUNIT_ASSERT_EQUAL(1UL, _cache->getMetrics().body_evictions.getValue()); -} - -void -MemCacheTest::testKeepBodyWhenLessThanOneFourth() -{ - setup(450); - - setSize(BucketId(16, 1), 150, 0, 0); - setSize(BucketId(16, 2), 100, 50, 50); - - stealMemory(150); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) \n" - "BucketId(0x4000000000000002) body,header\n"), - getBucketStatus(2)); -} - -void -MemCacheTest::testEvictHeader() -{ - setup(550); - - CPPUNIT_ASSERT_EQUAL(0UL, _cache->getMetrics().header_evictions.getValue()); - - setSize(BucketId(16, 1), 150, 0, 0); - setSize(BucketId(16, 2), 100, 200, 100); - - stealMemory(150); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) meta only\n" - "BucketId(0x4000000000000002) meta only\n"), - getBucketStatus(2)); - CPPUNIT_ASSERT_EQUAL(1UL, _cache->getMetrics().header_evictions.getValue()); -} - -#define ASSERT_CACHE_EVICTIONS(meta, header, body) \ - CPPUNIT_ASSERT_EQUAL(size_t(meta), _cache->getMetrics().body_evictions.getValue()); \ - CPPUNIT_ASSERT_EQUAL(size_t(header), _cache->getMetrics().header_evictions.getValue()); \ - CPPUNIT_ASSERT_EQUAL(size_t(body), _cache->getMetrics().meta_evictions.getValue()); - -void -MemCacheTest::testComplexEviction() -{ - setup(4200); - - setSize(BucketId(16, 1), 150, 0, 0); - setSize(BucketId(16, 2), 100, 200, 200); - setSize(BucketId(16, 3), 100, 200, 0); - setSize(BucketId(16, 4), 100, 400, 0); - setSize(BucketId(16, 5), 100, 200, 400); - setSize(BucketId(16, 6), 100, 200, 300); - setSize(BucketId(16, 7), 100, 0, 0); - setSize(BucketId(16, 8), 100, 200, 400); - setSize(BucketId(16, 9), 100, 200, 250); - - CPPUNIT_ASSERT_EQUAL(4100ul, cacheSize()); - - ASSERT_CACHE_EVICTIONS(0, 0, 0); - - stealMemory(600); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) meta only\n" - "BucketId(0x4000000000000002) header\n" - "BucketId(0x4000000000000003) header\n" - "BucketId(0x4000000000000004) header\n" - "BucketId(0x4000000000000005) header\n" - "BucketId(0x4000000000000006) body,header\n" - "BucketId(0x4000000000000007) meta only\n" - "BucketId(0x4000000000000008) body,header\n" - "BucketId(0x4000000000000009) body,header\n"), - getBucketStatus(9)); - - CPPUNIT_ASSERT_EQUAL(3500ul, cacheSize()); - - ASSERT_CACHE_EVICTIONS(2, 0, 0); - - stealMemory(500); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) meta only\n" - "BucketId(0x4000000000000002) meta only\n" - "BucketId(0x4000000000000003) meta only\n" - "BucketId(0x4000000000000004) header\n" - "BucketId(0x4000000000000005) header\n" - "BucketId(0x4000000000000006) body,header\n" - "BucketId(0x4000000000000007) meta only\n" - "BucketId(0x4000000000000008) body,header\n" - "BucketId(0x4000000000000009) body,header\n"), - getBucketStatus(9)); - - CPPUNIT_ASSERT_EQUAL(3100ul, cacheSize()); - - ASSERT_CACHE_EVICTIONS(2, 2, 0); - - stealMemory(1000); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) \n" - "BucketId(0x4000000000000002) meta only\n" - "BucketId(0x4000000000000003) meta only\n" - "BucketId(0x4000000000000004) meta only\n" - "BucketId(0x4000000000000005) meta only\n" - "BucketId(0x4000000000000006) header\n" - "BucketId(0x4000000000000007) meta only\n" - "BucketId(0x4000000000000008) body,header\n" - "BucketId(0x4000000000000009) body,header\n"), - getBucketStatus(9)); - - CPPUNIT_ASSERT_EQUAL(2050ul, cacheSize()); - - ASSERT_CACHE_EVICTIONS(3, 4, 1); - - stealMemory(1100); - - CPPUNIT_ASSERT_EQUAL( - std::string( - "BucketId(0x4000000000000001) \n" - "BucketId(0x4000000000000002) \n" - "BucketId(0x4000000000000003) \n" - "BucketId(0x4000000000000004) \n" - "BucketId(0x4000000000000005) \n" - "BucketId(0x4000000000000006) \n" - "BucketId(0x4000000000000007) meta only\n" - "BucketId(0x4000000000000008) header\n" - "BucketId(0x4000000000000009) body,header\n"), - getBucketStatus(9)); - - CPPUNIT_ASSERT_EQUAL(950ul, cacheSize()); -} - -#undef ASSERT_CACHE_EVICTIONS - -void -MemCacheTest::testEraseEmptyOnReturn() -{ - setup(4200); - setSize(BucketId(16, 1), 0, 0, 0); - CPPUNIT_ASSERT(!_cache->contains(document::BucketId(16, 1))); -} - -void -MemCacheTest::testDeleteDoesNotReAddMemoryUsage() -{ - BucketId id(16, 1); - setup(1000); - setSize(id, 100, 200, 300); - CPPUNIT_ASSERT_EQUAL(600ul, cacheSize()); - { - MemFilePtr file(_cache->get(id, env(), env().getDirectory())); - file.deleteFile(); - } - CPPUNIT_ASSERT_EQUAL(0ul, cacheSize()); - -} - -void -MemCacheTest::testGetWithNoCreation() -{ - BucketId id(16, 1); - setup(1000); - setSize(id, 100, 200, 300, false); - CPPUNIT_ASSERT_EQUAL(0ul, cacheSize()); -} - - -void -MemCacheTest::testEraseDoesNotReAddMemoryUsage() -{ - BucketId id(16, 1); - setup(1000); - setSize(id, 100, 200, 300); - CPPUNIT_ASSERT_EQUAL(600ul, cacheSize()); - { - MemFilePtr file(_cache->get(id, env(), env().getDirectory())); - file.eraseFromCache(); - } - CPPUNIT_ASSERT_EQUAL(0ul, cacheSize()); - -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/spi/memfileautorepairtest.cpp b/memfilepersistence/src/tests/spi/memfileautorepairtest.cpp deleted file mode 100644 index b7fbeba1649..00000000000 --- a/memfilepersistence/src/tests/spi/memfileautorepairtest.cpp +++ /dev/null @@ -1,409 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include -#include - -using storage::spi::test::makeSpiBucket; - -namespace storage { -namespace memfile { - -class MemFileAutoRepairTest : public SingleDiskMemFileTestUtils -{ -public: - void setUp() override; - void tearDown() override; - - void testFileMetadataCorruptionIsAutoRepaired(); - void testDocumentContentCorruptionIsAutoRepaired(); - void testCorruptionEvictsBucketFromCache(); - void testRepairFailureInMaintainEvictsBucketFromCache(); - void testZeroLengthFileIsDeleted(); - void testTruncatedBodyLocationIsAutoRepaired(); - void testTruncatedHeaderLocationIsAutoRepaired(); - void testTruncatedHeaderBlockIsAutoRepaired(); - - void corruptBodyBlock(); - - CPPUNIT_TEST_SUITE(MemFileAutoRepairTest); - CPPUNIT_TEST(testFileMetadataCorruptionIsAutoRepaired); - CPPUNIT_TEST(testDocumentContentCorruptionIsAutoRepaired); - CPPUNIT_TEST(testCorruptionEvictsBucketFromCache); - CPPUNIT_TEST(testRepairFailureInMaintainEvictsBucketFromCache); - CPPUNIT_TEST(testZeroLengthFileIsDeleted); - CPPUNIT_TEST(testTruncatedBodyLocationIsAutoRepaired); - CPPUNIT_TEST(testTruncatedHeaderLocationIsAutoRepaired); - CPPUNIT_TEST(testTruncatedHeaderBlockIsAutoRepaired); - CPPUNIT_TEST_SUITE_END(); - -private: - void assertDocumentIsSilentlyRemoved( - const document::BucketId& bucket, - const document::DocumentId& docId); - - void reconfigureMinimumHeaderBlockSize(uint32_t newMinSize); - - document::BucketId _bucket; - std::unique_ptr _file; - std::vector _slotIds; -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemFileAutoRepairTest); - -namespace { - // A totall uncached memfile with content to use for verify testing - std::unique_ptr _memFile; - - // Clear old content. Create new file. Make sure nothing is cached. - void prepareBucket(SingleDiskMemFileTestUtils& util, - const FileSpecification& file) { - _memFile.reset(); - util.env()._cache.clear(); - vespalib::unlink(file.getPath()); - util.createTestBucket(file.getBucketId(), 0); - util.env()._cache.clear(); - _memFile.reset(new MemFile(file, util.env())); - _memFile->getMemFileIO().close(); - - } - - MetaSlot getSlot(uint32_t index) { - assert(_memFile.get()); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - MetaSlot result; - file.read(&result, sizeof(MetaSlot), - sizeof(Header) + sizeof(MetaSlot) * index); - return result; - } - - void setSlot(uint32_t index, MetaSlot slot, - bool updateFileChecksum = true) - { - (void)updateFileChecksum; - assert(_memFile.get()); - //if (updateFileChecksum) slot.updateFileChecksum(); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - file.write(&slot, sizeof(MetaSlot), - sizeof(Header) + sizeof(MetaSlot) * index); - } -} - -void -MemFileAutoRepairTest::setUp() -{ - SingleDiskMemFileTestUtils::setUp(); - _bucket = BucketId(16, 0xa); - createTestBucket(_bucket, 0); - - { - MemFilePtr memFilePtr(env()._cache.get(_bucket, env(), env().getDirectory())); - _file.reset(new FileSpecification(memFilePtr->getFile())); - CPPUNIT_ASSERT(memFilePtr->getSlotCount() >= 2); - for (size_t i = 0; i < memFilePtr->getSlotCount(); ++i) { - _slotIds.push_back(memFilePtr->getDocumentId((*memFilePtr)[i])); - } - } - env()._cache.clear(); -} - -void -MemFileAutoRepairTest::tearDown() -{ - _file.reset(0); - _memFile.reset(0); - SingleDiskMemFileTestUtils::tearDown(); -}; - -void -MemFileAutoRepairTest::testFileMetadataCorruptionIsAutoRepaired() -{ - // Test corruption detected in initial metadata load - prepareBucket(*this, *_file); - document::DocumentId id(_slotIds[1]); - MetaSlot slot(getSlot(1)); - CPPUNIT_ASSERT(slot._gid == id.getGlobalId()); // Sanity checking... - { - MetaSlot s(slot); - s.setTimestamp(Timestamp(40)); - setSlot(1, s); - } - - CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets()); - - // File not in cache; should be detected in initial load - spi::GetResult res(doGet(_bucket, id, document::AllFields())); - // FIXME: currently loadFile is silently fixing corruptions! - //CPPUNIT_ASSERT_EQUAL(spi::Result::TRANSIENT_ERROR, res.getErrorCode()); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res.getErrorCode()); - CPPUNIT_ASSERT(!res.hasDocument()); - - CPPUNIT_ASSERT_EQUAL(std::string("400000000000000a"), getModifiedBuckets()); - CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets()); - - // File should now have been repaired, so a subsequent get for - // the same document should just return an empty (but OK) result. - spi::GetResult res2(doGet(_bucket, id, document::AllFields())); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res2.getErrorCode()); - CPPUNIT_ASSERT(!res2.hasDocument()); - - CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets()); -} - -void -MemFileAutoRepairTest::corruptBodyBlock() -{ - CPPUNIT_ASSERT(!env()._cache.contains(_bucket)); - // Corrupt body block of slot 1 - MetaSlot slot(getSlot(1)); - { - MetaSlot s(slot); - s.setBodyPos(52); - s.setBodySize(18); - s.updateChecksum(); - setSlot(1, s); - } -} - -void -MemFileAutoRepairTest::testDocumentContentCorruptionIsAutoRepaired() -{ - // Corrupt body block - prepareBucket(*this, *_file); - document::DocumentId id(_slotIds[1]); - corruptBodyBlock(); - - CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets()); - - spi::GetResult res(doGet(_bucket, id, document::AllFields())); - CPPUNIT_ASSERT_EQUAL(spi::Result::TRANSIENT_ERROR, res.getErrorCode()); - CPPUNIT_ASSERT(!res.hasDocument()); - - CPPUNIT_ASSERT(!env()._cache.contains(_bucket)); - - CPPUNIT_ASSERT_EQUAL(std::string("400000000000000a"), getModifiedBuckets()); - CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets()); - - // File should now have been repaired, so a subsequent get for - // the same document should just return an empty (but OK) result. - spi::GetResult res2(doGet(_bucket, id, document::AllFields())); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res2.getErrorCode()); - CPPUNIT_ASSERT(!res2.hasDocument()); - - // File should now be in cache OK - CPPUNIT_ASSERT(env()._cache.contains(_bucket)); - CPPUNIT_ASSERT_EQUAL(std::string(""), getModifiedBuckets()); -} - -// Ideally we'd test this for each spi operation that accesses MemFiles, but -// they all use the same eviction+auto-repair logic... -void -MemFileAutoRepairTest::testCorruptionEvictsBucketFromCache() -{ - prepareBucket(*this, *_file); - corruptBodyBlock(); - - // Read slot 0 and shove file into cache - spi::GetResult res(doGet(_bucket, _slotIds[0], document::AllFields())); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res.getErrorCode()); - CPPUNIT_ASSERT(res.hasDocument()); - CPPUNIT_ASSERT(env()._cache.contains(_bucket)); - - spi::GetResult res2(doGet(_bucket, _slotIds[1], document::AllFields())); - CPPUNIT_ASSERT_EQUAL(spi::Result::TRANSIENT_ERROR, res2.getErrorCode()); - CPPUNIT_ASSERT(!res2.hasDocument()); - - // Out of the cache! Begone! Shoo! - CPPUNIT_ASSERT(!env()._cache.contains(_bucket)); - -} - -void -MemFileAutoRepairTest::testRepairFailureInMaintainEvictsBucketFromCache() -{ - prepareBucket(*this, *_file); - corruptBodyBlock(); - spi::Result result(getPersistenceProvider().maintain( - makeSpiBucket(_bucket), spi::HIGH)); - // File being successfully repaired does not constitute a failure of - // the maintain() call. - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - // It should, however, shove it out of the cache. - CPPUNIT_ASSERT(!env()._cache.contains(_bucket)); -} - -void -MemFileAutoRepairTest::testZeroLengthFileIsDeleted() -{ - // Completely truncate auto-created file - vespalib::LazyFile file(_file->getPath(), 0); - file.resize(0); - - // No way to deal with zero-length files aside from deleting them. - spi::Result result(getPersistenceProvider().maintain( - makeSpiBucket(_bucket), spi::HIGH)); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, result.getErrorCode()); - CPPUNIT_ASSERT(!env()._cache.contains(_bucket)); - CPPUNIT_ASSERT(!vespalib::fileExists(_file->getPath())); -} - -namespace { - -uint32_t -alignDown(uint32_t value) -{ - uint32_t blocks = value / 512; - return blocks * 512; -}; - -FileInfo -fileInfoFromMemFile(const MemFilePtr& mf) -{ - auto& ioBuf(dynamic_cast( - mf->getMemFileIO())); - return ioBuf.getFileInfo(); -} - -} - -void -MemFileAutoRepairTest::assertDocumentIsSilentlyRemoved( - const document::BucketId& bucket, - const document::DocumentId& docId) -{ - // Corrupted (truncated) slot should be transparently removed during - // loadFile and it should be as if it was never there! - spi::Bucket spiBucket(makeSpiBucket(bucket)); - spi::GetResult res(doGet(spiBucket, docId, document::AllFields())); - CPPUNIT_ASSERT_EQUAL(spi::Result::NONE, res.getErrorCode()); - CPPUNIT_ASSERT(!res.hasDocument()); -} - -void -MemFileAutoRepairTest::testTruncatedBodyLocationIsAutoRepaired() -{ - document::BucketId bucket(16, 4); - document::Document::SP doc( - createRandomDocumentAtLocation(4, 1234, 1024, 1024)); - - doPut(doc, bucket, framework::MicroSecTime(1000)); - flush(bucket); - FileInfo fileInfo; - { - MemFilePtr mf(getMemFile(bucket)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), mf->getSlotCount()); - fileInfo = fileInfoFromMemFile(mf); - - const uint32_t bodyBlockStart( - sizeof(Header) - + fileInfo._metaDataListSize * sizeof(MetaSlot) - + fileInfo._headerBlockSize); - - vespalib::LazyFile file(mf->getFile().getPath(), 0); - uint32_t slotBodySize = (*mf)[0].getLocation(BODY)._size; - CPPUNIT_ASSERT(slotBodySize > 0); - // Align down to nearest sector alignment to avoid unrelated DirectIO - // checks to kick in. Since the body block is always aligned on a - // sector boundary, we know this cannot truncate into the header block. - file.resize(alignDown(bodyBlockStart + slotBodySize - 1)); - } - env()._cache.clear(); - assertDocumentIsSilentlyRemoved(bucket, doc->getId()); -} - -void -MemFileAutoRepairTest::testTruncatedHeaderLocationIsAutoRepaired() -{ - document::BucketId bucket(16, 4); - document::Document::SP doc( - createRandomDocumentAtLocation(4, 1234, 1024, 1024)); - // Ensure header has a bunch of data (see alignment comments below). - doc->setValue(doc->getField("hstringval"), - document::StringFieldValue(std::string(1024, 'A'))); - - doPut(doc, bucket, framework::MicroSecTime(1000)); - flush(bucket); - FileInfo fileInfo; - { - MemFilePtr mf(getMemFile(bucket)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), mf->getSlotCount()); - fileInfo = fileInfoFromMemFile(mf); - - const uint32_t headerBlockStart( - sizeof(Header) - + fileInfo._metaDataListSize * sizeof(MetaSlot)); - - vespalib::LazyFile file(mf->getFile().getPath(), 0); - uint32_t slotHeaderSize = (*mf)[0].getLocation(HEADER)._size; - CPPUNIT_ASSERT(slotHeaderSize > 0); - // Align down to nearest sector alignment to avoid unrelated DirectIO - // checks to kick in. The header block is not guaranteed to start on - // sector boundary, but we assume there is enough slack in the header - // section for the metadata slots themselves to be untouched since we - // have a minimum header size of 1024 for the doc in question. - file.resize(alignDown(headerBlockStart + slotHeaderSize - 1)); - } - env()._cache.clear(); - assertDocumentIsSilentlyRemoved(bucket, doc->getId()); -} - -void -MemFileAutoRepairTest::reconfigureMinimumHeaderBlockSize(uint32_t newMinSize) -{ - using MemFileConfig = vespa::config::storage::StorMemfilepersistenceConfig; - using MemFileConfigBuilder - = vespa::config::storage::StorMemfilepersistenceConfigBuilder; - MemFileConfigBuilder builder( - *env().acquireConfigReadLock().memFilePersistenceConfig()); - builder.minimumFileMetaSlots = 2; - builder.minimumFileHeaderBlockSize = newMinSize; - auto newConfig = std::unique_ptr(new MemFileConfig(builder)); - env().acquireConfigWriteLock().setMemFilePersistenceConfig( - std::move(newConfig)); -} - -void -MemFileAutoRepairTest::testTruncatedHeaderBlockIsAutoRepaired() -{ - document::BucketId bucket(16, 4); - document::Document::SP doc( - createRandomDocumentAtLocation(4, 1234, 1, 1)); - // Ensure header block is large enough that free space is added to the end. - reconfigureMinimumHeaderBlockSize(8192); - // Add header field and remove randomly generated body field, ensuring - // we have no data to add to body field. This will prevent slot body - // location checking from detecting a header truncation. - doc->setValue(doc->getField("hstringval"), - document::StringFieldValue("foo")); - doc->remove(doc->getField("content")); - - doPut(doc, bucket, framework::MicroSecTime(1000)); - flush(bucket); - FileInfo fileInfo; - { - MemFilePtr mf(getMemFile(bucket)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), mf->getSlotCount()); - fileInfo = fileInfoFromMemFile(mf); - - const uint32_t headerBlockEnd( - sizeof(Header) - + fileInfo._metaDataListSize * sizeof(MetaSlot) - + fileInfo._headerBlockSize); - - vespalib::LazyFile file(mf->getFile().getPath(), 0); - CPPUNIT_ASSERT_EQUAL(uint32_t(0), - (*mf)[0].getLocation(BODY)._size); // No body. - const auto headerLoc((*mf)[0].getLocation(HEADER)); - const uint32_t extent(headerLoc._pos + headerLoc._size); - // Make sure we don't intersect an existing slot range. - CPPUNIT_ASSERT(extent < alignDown(headerBlockEnd - 1)); - file.resize(alignDown(headerBlockEnd - 1)); - } - env()._cache.clear(); - assertDocumentIsSilentlyRemoved(bucket, doc->getId()); -} - -} -} diff --git a/memfilepersistence/src/tests/spi/memfiletest.cpp b/memfilepersistence/src/tests/spi/memfiletest.cpp deleted file mode 100644 index 019b20de2df..00000000000 --- a/memfilepersistence/src/tests/spi/memfiletest.cpp +++ /dev/null @@ -1,987 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace storage { -namespace memfile { - -struct MemFileTest : public SingleDiskMemFileTestUtils -{ - typedef MemFileCompactor::SlotList SlotList; - - /** - * Feed a document whose ID is deterministically generated from `seed` to - * bucket (16, 4) at time `timestamp`. - */ - document::DocumentId feedDocument( - uint64_t seed, - uint64_t timestamp, - uint32_t headerSize = 0, - uint32_t minBodySize = 10, - uint32_t maxBodySize = 100); - - /** - * Feed n instances of documents with the same ID to bucket (16, 4) using - * a timestamp range of [1000, 1000+n). - */ - void feedSameDocNTimes(uint32_t n); - - void setMaxDocumentVersionsOption(uint32_t n); - - std::vector compactWithVersionLimit(uint32_t maxVersions); - - void testCompactRemoveDoublePut(); - void testCompactPutRemove(); - void testCompactGidCollision(); - void testCompactGidCollisionAndNot(); - void testCompactWithMemFile(); - void testCompactCombined(); - void testCompactDifferentPuts(); - void testNoCompactionWhenDocumentVersionsWithinLimit(); - void testCompactWhenDocumentVersionsExceedLimit(); - void testCompactLimit1KeepsNewestVersionOnly(); - void testCompactionOptionsArePropagatedFromConfig(); - void testZeroDocumentVersionConfigIsCorrected(); - void testResizeToFreeSpace(); - void testNoFileWriteOnNoOpCompaction(); - void testCacheSize(); - void testClearCache(); - void testGetSlotsByTimestamp(); - void testCacheInconsistentSlot(); - void testEnsureCached(); - void testAddSlotWhenDiskFull(); - void testGetSerializedSize(); - void testGetBucketInfo(); - void testCopySlotsPreservesLocationSharing(); - void testFlushingToNonExistingFileAlwaysRunsCompaction(); - void testOrderDocSchemeDocumentsCanBeAddedToFile(); - - CPPUNIT_TEST_SUITE(MemFileTest); - CPPUNIT_TEST(testCompactRemoveDoublePut); - CPPUNIT_TEST(testCompactPutRemove); - CPPUNIT_TEST(testCompactGidCollision); - CPPUNIT_TEST(testCompactGidCollisionAndNot); - CPPUNIT_TEST(testCompactWithMemFile); - CPPUNIT_TEST(testCompactCombined); - CPPUNIT_TEST(testCompactDifferentPuts); - CPPUNIT_TEST(testNoCompactionWhenDocumentVersionsWithinLimit); - CPPUNIT_TEST(testCompactWhenDocumentVersionsExceedLimit); - CPPUNIT_TEST(testCompactLimit1KeepsNewestVersionOnly); - CPPUNIT_TEST(testCompactionOptionsArePropagatedFromConfig); - CPPUNIT_TEST(testZeroDocumentVersionConfigIsCorrected); - CPPUNIT_TEST(testNoFileWriteOnNoOpCompaction); - CPPUNIT_TEST(testCacheSize); - CPPUNIT_TEST(testClearCache); - CPPUNIT_TEST(testGetSlotsByTimestamp); - CPPUNIT_TEST(testEnsureCached); - CPPUNIT_TEST(testResizeToFreeSpace); - CPPUNIT_TEST(testAddSlotWhenDiskFull); - CPPUNIT_TEST(testGetSerializedSize); - CPPUNIT_TEST(testGetBucketInfo); - CPPUNIT_TEST(testCopySlotsPreservesLocationSharing); - CPPUNIT_TEST(testFlushingToNonExistingFileAlwaysRunsCompaction); - CPPUNIT_TEST(testOrderDocSchemeDocumentsCanBeAddedToFile); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemFileTest); - -/** - * Slots should actually be the same pointer. Use this assert to do correct - * check, and still print content of slots on failure. - */ -#define ASSERT_SLOT_EQUAL(slotptra, slotptrb) \ -{ \ - CPPUNIT_ASSERT(slotptra != 0); \ - CPPUNIT_ASSERT(slotptrb != 0); \ - std::ostringstream slotdiff; \ - slotdiff << "Expected: " << *slotptra << ", but got " << *slotptrb; \ - CPPUNIT_ASSERT_EQUAL_MSG(slotdiff.str(), slotptra, slotptrb); \ -} - -namespace { - -framework::MicroSecTime sec(uint64_t n) { - return framework::MicroSecTime(n * 1000000ULL); -} - -/** - * Utility functions for tests to call to do compacting, such that the - * tests themselves are not bound to the current interface. - * - * Also, this function translates second time to microsecond time. - */ -MemFileTest::SlotList getSlotsToRemove( - const MemFile& file, uint64_t currentTime, - uint64_t revertTime, uint64_t keepRemoveTime) -{ - MemFileCompactor compactor( - sec(currentTime), - CompactionOptions() - .maxDocumentVersions( - std::numeric_limits::max()) - .revertTimePeriod(sec(revertTime)) - .keepRemoveTimePeriod(sec(keepRemoveTime))); - return compactor.getSlotsToRemove(file); -} - -class AutoFlush -{ -public: - AutoFlush(MemFilePtr& ptr) : _ptr(ptr) {} - ~AutoFlush() { _ptr->flushToDisk(); } -private: - MemFilePtr& _ptr; -}; - -} - -document::DocumentId -MemFileTest::feedDocument( - uint64_t seed, - uint64_t timestamp, - uint32_t headerSize, - uint32_t minDocSize, - uint32_t maxDocSize) { - document::Document::SP doc(createRandomDocumentAtLocation( - 4, seed, minDocSize, maxDocSize)); - - if (headerSize > 0) { - std::string val(headerSize, 'A'); - doc->setValue(doc->getField("hstringval"), - document::StringFieldValue(val)); - } - - doPut(doc, - document::BucketId(16, 4), - Timestamp(timestamp * 1000000)); - - return doc->getId(); -} - -void -MemFileTest::feedSameDocNTimes(uint32_t n) -{ - for (uint32_t i = 0; i < n; ++i) { - feedDocument(1234, 1000 + i); - } -} - -void -MemFileTest::setMaxDocumentVersionsOption(uint32_t n) -{ - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options) - .maxDocumentVersions(n) - .build()); -} - -void -MemFileTest::testCacheSize() -{ - // Feed some puts - for (uint32_t i = 0; i < 4; i++) { - feedDocument(1234 * (i % 2), 1000 + 200 * i); - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - - CPPUNIT_ASSERT(file->getCacheSize().sum() > 0); -} - -void -MemFileTest::testClearCache() -{ - // Feed some puts - for (uint32_t i = 0; i < 4; i++) { - feedDocument(1234 * (i % 2), 1000 + 200 * i); - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - file->flushToDisk(); - - CPPUNIT_ASSERT(file->getCacheSize().bodySize > 0); - CPPUNIT_ASSERT(file->getCacheSize().headerSize > 0); - - file->clearCache(HEADER); - - CPPUNIT_ASSERT(file->getCacheSize().bodySize > 0); - CPPUNIT_ASSERT(file->getMemFileIO().getCachedSize(BODY) > 0); - CPPUNIT_ASSERT_EQUAL(0, (int)file->getCacheSize().headerSize); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), file->getMemFileIO().getCachedSize(HEADER)); - - file->clearCache(BODY); - - CPPUNIT_ASSERT_EQUAL(0, (int)file->getCacheSize().bodySize); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), file->getMemFileIO().getCachedSize(BODY)); -} - - -void -MemFileTest::testCompactGidCollision() -{ - // Feed two puts - for (uint32_t i = 0; i < 2; i++) { - feedDocument(1234 * i, 1000 + 200 * i); - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - const_cast((*file)[1]).setGlobalId((*file)[0].getGlobalId()); - - CPPUNIT_ASSERT_EQUAL(2, (int)file->getSlotCount()); - - { - SlotList toRemove(getSlotsToRemove(*file, 1600, 300, 86400)); - CPPUNIT_ASSERT_EQUAL(0, (int)toRemove.size()); - file->removeSlots(toRemove); - } -} - -void -MemFileTest::testCompactGidCollisionAndNot() -{ - // Feed some puts - for (uint32_t i = 0; i < 4; i++) { - feedDocument(1234 * (i % 2), 1000 + 200 * i); - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - const_cast((*file)[2]).setGlobalId((*file)[0].getGlobalId()); - const_cast((*file)[3]).setGlobalId((*file)[1].getGlobalId()); - - CPPUNIT_ASSERT_EQUAL(4, (int)file->getSlotCount()); - - { - SlotList toRemove(getSlotsToRemove(*file, 2000, 300, 86400)); - - CPPUNIT_ASSERT_EQUAL(2, (int)toRemove.size()); - ASSERT_SLOT_EQUAL(&(*file)[0], toRemove[0]); - ASSERT_SLOT_EQUAL(&(*file)[1], toRemove[1]); - file->removeSlots(toRemove); - } -} - - -void -MemFileTest::testCompactRemoveDoublePut() -{ - // Feed two puts at time 1000 and 1200 - for (uint32_t i = 0; i < 2; i++) { - feedDocument(1234, 1000 + 200 * i); - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - CPPUNIT_ASSERT_EQUAL(2, (int)file->getSlotCount()); - - { - // Not time to collect yet, newest is still revertable - SlotList toRemove(getSlotsToRemove(*file, 1300, 300, 86400)); - CPPUNIT_ASSERT_EQUAL(0, (int)toRemove.size()); - } - - { - SlotList toRemove(getSlotsToRemove(*file, 1600, 300, 86400)); - - CPPUNIT_ASSERT_EQUAL(1, (int)toRemove.size()); - ASSERT_SLOT_EQUAL(&(*file)[0], toRemove[0]); - file->removeSlots(toRemove); - } -} - -void -MemFileTest::testCompactPutRemove() -{ - document::DocumentId docId = feedDocument(1234, 1000); - - doRemove(docId, Timestamp(1200*1000000), 0); - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - - { - // Since remove can still be reverted, we can't revert anything. - SlotList toRemove(getSlotsToRemove(*file, 1300, 300, 600)); - - CPPUNIT_ASSERT_EQUAL(0, (int)toRemove.size()); - } - - { - SlotList toRemove(getSlotsToRemove(*file, 1600, 300, 600)); - - CPPUNIT_ASSERT_EQUAL(1, (int)toRemove.size()); - ASSERT_SLOT_EQUAL(&(*file)[0], toRemove[0]); - file->removeSlots(toRemove); - } - - { - SlotList toRemove(getSlotsToRemove(*file, 1900, 300, 600)); - - CPPUNIT_ASSERT_EQUAL(1, (int)toRemove.size()); - ASSERT_SLOT_EQUAL(&(*file)[0], toRemove[0]); - file->removeSlots(toRemove); - } -} - -void -MemFileTest::testCompactCombined() -{ - document::DocumentId docId; - - // Feed some puts at time 1000, 1200, 1400, 1600 and 1800 for same doc. - for (uint32_t i = 0; i < 5; i++) { - docId = feedDocument(1234, 1000 + i * 200); - } - flush(document::BucketId(16, 4)); - - // Now add remove at time 2000. - doRemove(docId, Timestamp(2000 * 1000000), 0); - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - CPPUNIT_ASSERT_EQUAL(6, (int)file->getSlotCount()); - - { - // Compact all redundant slots that are older than revert period of 300. - // This includes 1000, 1200, 1400 and 1600. - SlotList toRemove(getSlotsToRemove(*file, 2001, 300, 86400)); - CPPUNIT_ASSERT_EQUAL(4, (int)toRemove.size()); - for (int i = 0; i < 4; ++i) { - ASSERT_SLOT_EQUAL(&(*file)[i], toRemove[i]); - } - file->removeSlots(toRemove); - } -} - -void -MemFileTest::testCompactDifferentPuts() -{ - document::DocumentId docId; - - // Feed some puts - for (uint32_t i = 0; i < 2; i++) { - for (uint32_t j = 0; j < 3; j++) { - feedDocument(1234 * j, 1000 + (i * 3 + j) * 200); - } - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - CPPUNIT_ASSERT_EQUAL(6, (int)file->getSlotCount()); - - { - SlotList toRemove(getSlotsToRemove(*file, 3000, 300, 86400)); - CPPUNIT_ASSERT_EQUAL(3, (int)toRemove.size()); - - for (uint32_t i = 0; i < 3; i++) { - bool found = false; - for (uint32_t j = 0; j < 3; j++) { - if ((*file)[j] == *toRemove[i]) { - found = true; - } - } - - CPPUNIT_ASSERT(found); - } - file->removeSlots(toRemove); - } -} - -void -MemFileTest::testCompactWithMemFile() -{ - // Feed two puts - for (uint32_t i = 0; i < 2; i++) { - document::Document::SP doc(createRandomDocumentAtLocation( - 4, 1234, 10, 100)); - - doPut(doc, document::BucketId(16, 4), Timestamp((1000 + i * 200)*1000000), 0); - } - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - CPPUNIT_ASSERT_EQUAL(2, (int)file->getSlotCount()); - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options) - .revertTimePeriod(framework::MicroSecTime(1000)) - .build()); - - getFakeClock()._absoluteTime = framework::MicroSecTime(2000ULL * 1000000); - - CPPUNIT_ASSERT(file->compact()); - CPPUNIT_ASSERT(!file->compact()); - - CPPUNIT_ASSERT_EQUAL(1, (int)file->getSlotCount()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1200 * 1000000), (*file)[0].getTimestamp()); -} - -/** - * Feed 5 versions of a single document at absolute times 0 through 4 seconds - * and run compaction using the provided max document version option. - * Revert time/keep remove time options are effectively disabled for this test. - * Returns timestamps of all slots that are marked as compactable. - */ -std::vector -MemFileTest::compactWithVersionLimit(uint32_t maxVersions) -{ - document::BucketId bucket(16, 4); - std::shared_ptr doc( - createRandomDocumentAtLocation(4, 1234, 10, 100)); - uint32_t versionLimit = 5; - for (uint32_t i = 0; i < versionLimit; ++i) { - Timestamp ts(sec(i).getTime()); - doPut(doc, bucket, ts, 0); - } - flush(bucket); - - MemFilePtr file(getMemFile(bucket)); - CPPUNIT_ASSERT_EQUAL(versionLimit, file->getSlotCount()); - - framework::MicroSecTime currentTime(sec(versionLimit)); - MemFileCompactor compactor( - currentTime, - CompactionOptions() - .revertTimePeriod(sec(versionLimit)) - .keepRemoveTimePeriod(sec(versionLimit)) - .maxDocumentVersions(maxVersions)); - auto slots = compactor.getSlotsToRemove(*file); - // Convert to timestamps since caller won't have access to actual MemFile. - std::vector timestamps; - for (const MemSlot* slot : slots) { - timestamps.push_back(slot->getTimestamp()); - } - return timestamps; -} - -void -MemFileTest::testNoCompactionWhenDocumentVersionsWithinLimit() -{ - auto timestamps = compactWithVersionLimit(5); - CPPUNIT_ASSERT(timestamps.empty()); -} - -void -MemFileTest::testCompactWhenDocumentVersionsExceedLimit() -{ - auto timestamps = compactWithVersionLimit(2); - CPPUNIT_ASSERT_EQUAL(size_t(3), timestamps.size()); - std::vector expected = { - sec(0), sec(1), sec(2) - }; - CPPUNIT_ASSERT_EQUAL(expected, timestamps); -} - -void -MemFileTest::testCompactLimit1KeepsNewestVersionOnly() -{ - auto timestamps = compactWithVersionLimit(1); - CPPUNIT_ASSERT_EQUAL(size_t(4), timestamps.size()); - std::vector expected = { - sec(0), sec(1), sec(2), sec(3) - }; - CPPUNIT_ASSERT_EQUAL(expected, timestamps); -} - -void -MemFileTest::testCompactionOptionsArePropagatedFromConfig() -{ - vespa::config::storage::StorMemfilepersistenceConfigBuilder mfcBuilder; - vespa::config::content::PersistenceConfigBuilder pcBuilder; - - pcBuilder.maximumVersionsOfSingleDocumentStored = 12345; - pcBuilder.revertTimePeriod = 555; - pcBuilder.keepRemoveTimePeriod = 777; - - vespa::config::storage::StorMemfilepersistenceConfig mfc(mfcBuilder); - vespa::config::content::PersistenceConfig pc(pcBuilder); - Options opts(mfc, pc); - - CPPUNIT_ASSERT_EQUAL(framework::MicroSecTime(555 * 1000000), - opts._revertTimePeriod); - CPPUNIT_ASSERT_EQUAL(framework::MicroSecTime(777 * 1000000), - opts._keepRemoveTimePeriod); - CPPUNIT_ASSERT_EQUAL(uint32_t(12345), opts._maxDocumentVersions); -} - -void -MemFileTest::testZeroDocumentVersionConfigIsCorrected() -{ - vespa::config::storage::StorMemfilepersistenceConfigBuilder mfcBuilder; - vespa::config::content::PersistenceConfigBuilder pcBuilder; - - pcBuilder.maximumVersionsOfSingleDocumentStored = 0; - - vespa::config::storage::StorMemfilepersistenceConfig mfc(mfcBuilder); - vespa::config::content::PersistenceConfig pc(pcBuilder); - Options opts(mfc, pc); - - CPPUNIT_ASSERT_EQUAL(uint32_t(1), opts._maxDocumentVersions); -} - -void -MemFileTest::testGetSlotsByTimestamp() -{ - for (uint32_t i = 0; i < 10; i++) { - feedDocument(i, 1000 + i); - } - flush(document::BucketId(16, 4)); - - std::vector timestamps; - timestamps.push_back(Timestamp(999 * 1000000)); - timestamps.push_back(Timestamp(1001 * 1000000)); - timestamps.push_back(Timestamp(1002 * 1000000)); - timestamps.push_back(Timestamp(1007 * 1000000)); - timestamps.push_back(Timestamp(1100 * 1000000)); - std::vector slots; - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - file->getSlotsByTimestamp(timestamps, slots); - CPPUNIT_ASSERT_EQUAL(std::size_t(3), slots.size()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1001 * 1000000), slots[0]->getTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1002 * 1000000), slots[1]->getTimestamp()); - CPPUNIT_ASSERT_EQUAL(Timestamp(1007 * 1000000), slots[2]->getTimestamp()); -} - -void -MemFileTest::testEnsureCached() -{ - // Feed some puts - for (uint32_t i = 0; i < 5; i++) { - feedDocument(i, 1000 + i * 200, 600, 600, 600); - } - flush(document::BucketId(16, 4)); - - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options).maximumReadThroughGap(512).build()); - env()._cache.clear(); - - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - CPPUNIT_ASSERT(file.get()); - CPPUNIT_ASSERT_EQUAL(5, (int)file->getSlotCount()); - - file->ensureDocumentIdCached((*file)[1]); - - for (std::size_t i = 0; i < file->getSlotCount(); ++i) { - if (i == 1) { - CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i])); - } else { - CPPUNIT_ASSERT(!file->documentIdAvailable((*file)[i])); - } - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY)); - } - } - - env()._cache.clear(); - - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - file->ensureDocumentCached((*file)[2], true); - - for (std::size_t i = 0; i < file->getSlotCount(); ++i) { - if (i == 2) { - CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER)); - } else { - CPPUNIT_ASSERT(!file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], HEADER)); - } - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY)); - } - } - - env()._cache.clear(); - - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - - file->ensureDocumentCached((*file)[3], false); - - for (std::size_t i = 0; i < file->getSlotCount(); ++i) { - if (i == 3) { - CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER)); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], BODY)); - } else { - CPPUNIT_ASSERT(!file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], HEADER)); - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY)); - } - } - } - - env()._cache.clear(); - - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - - std::vector ts; - for (int i = 2; i < 5; ++i) { - ts.push_back((*file)[i].getTimestamp()); - } - - file->ensureDocumentCached(ts, false); - - for (std::size_t i = 0; i < file->getSlotCount(); ++i) { - if (i > 1 && i < 5) { - CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER)); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], BODY)); - } else { - CPPUNIT_ASSERT(!file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], HEADER)); - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY)); - } - } - } - - env()._cache.clear(); - - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - - file->ensureHeaderBlockCached(); - - for (std::size_t i = 0; i < file->getSlotCount(); ++i) { - CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER)); - CPPUNIT_ASSERT(!file->partAvailable((*file)[i], BODY)); - } - } - - env()._cache.clear(); - - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - - file->ensureBodyBlockCached(); - - for (std::size_t i = 0; i < file->getSlotCount(); ++i) { - CPPUNIT_ASSERT(file->documentIdAvailable((*file)[i])); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], HEADER)); - CPPUNIT_ASSERT(file->partAvailable((*file)[i], BODY)); - } - } -} - -void -MemFileTest::testResizeToFreeSpace() -{ - /** - * This test tests that files are resized to a smaller size when they need - * to be. This should happen during a call to flushToDisk() in MemFile, - * which is either dirty or if passed flag to check even if clean. (Which - * the integrity checker cycle uses). A clean file is used for testing to - * ensure that no part of the code only works for dirty files. This test - * only test for the case where body block is too large. The real - * implementation here will be in the flushUpdatesToFile() function for the - * given file formats. (VersionSerializer's) If more cases wants to be - * tested add those as unit tests for the versionserializers themselves. - */ - - // Create a test bucket to test with. - BucketId bucket(16, 0xa); - createTestBucket(bucket, 0); - - off_t file_size = - ((SimpleMemFileIOBuffer&)getMemFile(bucket)->getMemFileIO()). - getFileHandle().getFileSize(); - - // Clear cache so we can manually modify backing file to increase the - // size of it. - FileSpecification file(getMemFile(bucket)->getFile()); - env()._cache.clear(); - { - // Extend file to 1 MB, which should create an excessively large - // body block such that file should be resized to be smaller - vespalib::LazyFile fileHandle(file.getPath(), 0); - fileHandle.write("foobar", 6, 2 * 1024 * 1024 - 6); - } - MemFilePtr memFile(getMemFile(bucket)); - memFile->flushToDisk(CHECK_NON_DIRTY_FILE_FOR_SPACE); - CPPUNIT_ASSERT_EQUAL(file_size, - ((SimpleMemFileIOBuffer&)memFile->getMemFileIO()). - getFileHandle().getFileSize()); -} - -namespace { - -const vespalib::LazyFile& -getFileHandle(const MemFile& mf1) -{ - return dynamic_cast( - mf1.getMemFileIO()).getFileHandle(); -} - -const LoggingLazyFile& -getLoggerFile(const MemFile& file) -{ - return dynamic_cast(getFileHandle(file)); -} - -} - -void -MemFileTest::testNoFileWriteOnNoOpCompaction() -{ - BucketId bucket(16, 4); - env()._lazyFileFactory = std::unique_ptr( - new LoggingLazyFile::Factory()); - - // Feed some unique puts, none of which can be compacted away. - for (uint32_t i = 0; i < 2; i++) { - document::Document::SP doc(createRandomDocumentAtLocation( - 4, i, 10, 100)); - - doPut(doc, bucket, Timestamp((1000 + i * 200)*1000000), 0); - } - flush(bucket); - - MemFilePtr file(getMemFile(bucket)); - - size_t opsBeforeFlush = getLoggerFile(*file).getOperationCount(); - file->flushToDisk(CHECK_NON_DIRTY_FILE_FOR_SPACE); - size_t opsAfterFlush = getLoggerFile(*file).getOperationCount(); - - // Disk should not have been touched, since no slots have been - // compacted away. - if (opsBeforeFlush != opsAfterFlush) { - std::cerr << "\n" << getLoggerFile(*file).toString() << "\n"; - } - CPPUNIT_ASSERT_EQUAL(opsBeforeFlush, opsAfterFlush); -} - -void -MemFileTest::testAddSlotWhenDiskFull() -{ - { - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - { - // Add a dummy-slot that can later be removed - Document::SP doc(createRandomDocumentAtLocation(4)); - file->addPutSlot(*doc, Timestamp(1001)); - } - } - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - AutoFlush af(file); - PartitionMonitor* mon = env().getDirectory().getPartition().getMonitor(); - // Set disk to 99% full - mon->setStatOncePolicy(); - mon->setMaxFillness(.98f); - mon->overrideRealStat(512, 100000, 99000); - CPPUNIT_ASSERT(mon->isFull()); - - // Test that addSlot with a non-persisted Put fails - { - Document::SP doc(createRandomDocumentAtLocation(4)); - try { - file->addPutSlot(*doc, Timestamp(10003)); - CPPUNIT_ASSERT(false); - } catch (vespalib::IoException& e) { - CPPUNIT_ASSERT_EQUAL(vespalib::IoException::NO_SPACE, e.getType()); - } - } - - // Slots with valid header and body locations should also - // not fail, as these are added when the file is loaded - { - // Just steal parts from existing slot to ensure they're persisted - const MemSlot* existing = file->getSlotAtTime(Timestamp(1001)); - - MemSlot slot(existing->getGlobalId(), - Timestamp(1005), - existing->getLocation(HEADER), - existing->getLocation(BODY), - IN_USE, - 0x1234); - file->addSlot(slot); - } - - // Removes should not fail when disk is full - { - file->addRemoveSlot(*file->getSlotAtTime(Timestamp(1001)), Timestamp(1003)); - } -} - -void -MemFileTest::testGetSerializedSize() { - document::Document::SP doc(createRandomDocumentAtLocation( - 4, 1234, 1024, 1024)); - - std::string val("Header"); - doc->setValue(doc->getField("hstringval"), - document::StringFieldValue(val)); - - doPut(doc, document::BucketId(16, 4), framework::MicroSecTime(1000)); - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - file->ensureBodyBlockCached(); - const MemSlot* slot = file->getSlotAtTime(framework::MicroSecTime(1000)); - CPPUNIT_ASSERT(slot != 0); - - vespalib::nbostream serializedHeader; - doc->serializeHeader(serializedHeader); - - vespalib::nbostream serializedBody; - doc->serializeBody(serializedBody); - - CPPUNIT_ASSERT_EQUAL(uint32_t(serializedHeader.size()), - file->getSerializedSize(*slot, HEADER)); - CPPUNIT_ASSERT_EQUAL(uint32_t(serializedBody.size()), - file->getSerializedSize(*slot, BODY)); -} - -void -MemFileTest::testGetBucketInfo() -{ - document::Document::SP doc(createRandomDocumentAtLocation( - 4, 1234, 100, 100)); - doc->setValue(doc->getField("content"), - document::StringFieldValue("foo")); - document::Document::SP doc2(createRandomDocumentAtLocation( - 4, 1235, 100, 100)); - doc2->setValue(doc->getField("content"), - document::StringFieldValue("bar")); - - doPut(doc, document::BucketId(16, 4), framework::MicroSecTime(1000)); - flush(document::BucketId(16, 4)); - - doPut(doc2, document::BucketId(16, 4), framework::MicroSecTime(1001)); - flush(document::BucketId(16, 4)); - - // Do remove which should only add a single meta entry - doRemove(doc->getId(), Timestamp(1002), 0); - flush(document::BucketId(16, 4)); - - MemFilePtr file(getMemFile(document::BucketId(16, 4))); - - CPPUNIT_ASSERT_EQUAL(3u, file->getSlotCount()); - uint32_t maxHeaderExtent = (*file)[1].getLocation(HEADER)._pos - + (*file)[1].getLocation(HEADER)._size; - uint32_t maxBodyExtent = (*file)[1].getLocation(BODY)._pos - + (*file)[1].getLocation(BODY)._size; - - uint32_t wantedUsedSize = 64 + 40*3 + maxHeaderExtent + maxBodyExtent; - BucketInfo info = file->getBucketInfo(); - CPPUNIT_ASSERT_EQUAL(1u, info.getDocumentCount()); - CPPUNIT_ASSERT_EQUAL(3u, info.getEntryCount()); - CPPUNIT_ASSERT_EQUAL(wantedUsedSize, info.getUsedSize()); - uint32_t wantedUniqueSize = (*file)[1].getLocation(HEADER)._size - + (*file)[1].getLocation(BODY)._size; - CPPUNIT_ASSERT_EQUAL(wantedUniqueSize, info.getDocumentSize()); -} - -void -MemFileTest::testCopySlotsPreservesLocationSharing() -{ - document::BucketId bucket(16, 4); - // Feed two puts to same document (identical seed). These should not - // share any blocks. Note: implicit sec -> microsec conversion. - feedDocument(1234, 1000); // slot 0 - auto docId = feedDocument(1234, 1001); // slot 1 - // Update only header of last version of document. This should share - // slot body block 2 with that slot 1. - auto update = createHeaderUpdate(docId, document::IntFieldValue(5678)); - doUpdate(bucket, update, Timestamp(1002 * 1000000), 0); - // Feed a remove for doc in slot 2. This should share the header block of - // slot 3 with the newest document in slot 2. - doRemove(docId, Timestamp(1003 * 1000000), 0); - flush(bucket); - - { - MemFilePtr src(getMemFile(document::BucketId(16, 4))); - MemFilePtr dest(getMemFile(document::BucketId(17, 4))); - std::vector timestamps { - Timestamp(1000 * 1000000), - Timestamp(1001 * 1000000), - Timestamp(1002 * 1000000), - Timestamp(1003 * 1000000) - }; - std::vector slots { - src->getSlotAtTime(Timestamp(1000 * 1000000)), - src->getSlotAtTime(Timestamp(1001 * 1000000)), - src->getSlotAtTime(Timestamp(1002 * 1000000)), - src->getSlotAtTime(Timestamp(1003 * 1000000)) - }; - dest->copySlotsFrom(*src, slots); - dest->flushToDisk(); - CPPUNIT_ASSERT_EQUAL(uint32_t(4), dest->getSlotCount()); - - DataLocation header[4]; - DataLocation body[4]; - for (int i = 0; i < 4; ++i) { - const MemSlot* slot = dest->getSlotAtTime(timestamps[i]); - header[i] = slot->getLocation(HEADER); - body[i] = slot->getLocation(BODY); - } - CPPUNIT_ASSERT(!(header[0] == header[1])); - - CPPUNIT_ASSERT_EQUAL(body[2], body[1]); - CPPUNIT_ASSERT_EQUAL(header[3], header[2]); - } -} - -void -MemFileTest::testFlushingToNonExistingFileAlwaysRunsCompaction() -{ - document::BucketId bucket(16, 4); - - setMaxDocumentVersionsOption(1); - feedSameDocNTimes(10); - flush(bucket); - - // Max version limit is 1, flushing should have compacted it down. - MemFilePtr file(getMemFile(bucket)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount()); -} - -void -MemFileTest::testOrderDocSchemeDocumentsCanBeAddedToFile() -{ - // Quick explanation of the esoteric and particular values chosen below: - // orderdoc mangles the MSB of the bucket ID based on the document ID's - // ordering parameters and thus its bucket cannot be directly deduced from - // the generated GID. The values given here specify a document whose GID - // bits differ from those generated by the document and where a GID-only - // bucket ownership check would fail (nuking the node with an assertion). - // We have to make sure cases do not trigger false positives. - document::BucketId bucket(0x84000000ee723751); - auto doc = createDocument("the quick red fox trips over a hedge", - "orderdoc(3,1):storage_test:group1:9:9"); - doPut(std::shared_ptr(std::move(doc)), - bucket, - Timestamp(1000000 * 1234)); - flush(bucket); - - MemFilePtr file(getMemFile(bucket)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), file->getSlotCount()); - // Ideally we'd test the failure case as well, but that'd require framework - // support for death tests. -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.cpp b/memfilepersistence/src/tests/spi/memfiletestutils.cpp deleted file mode 100644 index 9571d880e9f..00000000000 --- a/memfilepersistence/src/tests/spi/memfiletestutils.cpp +++ /dev/null @@ -1,455 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using document::DocumentType; -using document::test::makeBucketSpace; -using storage::spi::test::makeSpiBucket; - -namespace storage { -namespace memfile { - -namespace { - spi::LoadType defaultLoadType(0, "default"); -} - -namespace { - vdstestlib::DirConfig initialize(uint32_t numDisks) { - system(vespalib::make_string("rm -rf vdsroot").c_str()); - for (uint32_t i = 0; i < numDisks; i++) { - system(vespalib::make_string("mkdir -p vdsroot/disks/d%d", i).c_str()); - } - vdstestlib::DirConfig config(getStandardConfig(true)); - return config; - } - - template - struct ConfigReader : public T::Subscriber - { - T config; - - ConfigReader(const std::string& configId) { - T::subscribe(configId, *this); - } - void configure(const T& c) { config = c; } - }; -} - -MemFileTestEnvironment::MemFileTestEnvironment( - uint32_t numDisks, - framework::ComponentRegister& reg, - const document::DocumentTypeRepo& repo) - : _config(initialize(numDisks)), - _provider(reg, _config.getConfigId()) -{ - _provider.setDocumentRepo(repo); - _provider.getPartitionStates(); -} - -MemFileTestUtils::MemFileTestUtils() -{ -} - -MemFileTestUtils::~MemFileTestUtils() -{ -} - -void -MemFileTestUtils::setupDisks(uint32_t numDisks) { - tearDown(); - _componentRegister.reset( - new framework::defaultimplementation::ComponentRegisterImpl); - _clock.reset(new FakeClock); - _componentRegister->setClock(*_clock); - _env.reset(new MemFileTestEnvironment(numDisks, - *_componentRegister, - *getTypeRepo())); -} - -Environment& -MemFileTestUtils::env() -{ - return static_cast( - getPersistenceProvider()).getEnvironment(); -} - -MemFilePersistenceProvider& -MemFileTestUtils::getPersistenceProvider() -{ - return _env->_provider; -} - -MemFilePersistenceThreadMetrics& -MemFileTestUtils::getMetrics() -{ - return getPersistenceProvider().getMetrics(); -} - -std::string -MemFileTestUtils::getMemFileStatus(const document::BucketId& id, - uint32_t disk) -{ - MemFilePtr file(getMemFile(id, disk)); - std::ostringstream ost; - ost << id << ": " << file->getSlotCount() << "," << file->getDisk(); - return ost.str(); -} - -std::string -MemFileTestUtils::getModifiedBuckets() -{ - spi::BucketIdListResult result( - getPersistenceProvider().getModifiedBuckets(makeBucketSpace())); - const spi::BucketIdListResult::List& list(result.getList()); - std::ostringstream ss; - for (size_t i = 0; i < list.size(); ++i) { - if (i != 0) { - ss << ","; - } - ss << std::hex << list[i].getId(); - } - return ss.str(); -} - -MemFilePtr -MemFileTestUtils::getMemFile(const document::BucketId& id, uint16_t disk) -{ - return env()._cache.get(id, env(), env().getDirectory(disk)); -} - -spi::Result -MemFileTestUtils::flush(const document::BucketId& id, uint16_t disk) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - return getPersistenceProvider().flush( - makeSpiBucket(id, spi::PartitionId(disk)), context); -} - -document::Document::SP -MemFileTestUtils::doPutOnDisk( - uint16_t disk, - uint32_t location, - Timestamp timestamp, - uint32_t minSize, - uint32_t maxSize) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::Document::SP doc(createRandomDocumentAtLocation( - location, timestamp.getTime(), minSize, maxSize)); - getPersistenceProvider().put( - makeSpiBucket(document::BucketId(16, location), spi::PartitionId(disk)), - spi::Timestamp(timestamp.getTime()), - doc, - context); - return doc; -} - -bool -MemFileTestUtils::doRemoveOnDisk( - uint16_t disk, - const document::BucketId& bucketId, - const document::DocumentId& docId, - Timestamp timestamp, - OperationHandler::RemoveType persistRemove) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - if (persistRemove == OperationHandler::PERSIST_REMOVE_IF_FOUND) { - spi::RemoveResult result = getPersistenceProvider().removeIfFound( - makeSpiBucket(bucketId, spi::PartitionId(disk)), - spi::Timestamp(timestamp.getTime()), - docId, - context); - return result.wasFound(); - } - spi::RemoveResult result = getPersistenceProvider().remove( - makeSpiBucket(bucketId, spi::PartitionId(disk)), - spi::Timestamp(timestamp.getTime()), - docId, - context); - - return result.wasFound(); -} - -bool -MemFileTestUtils::doUnrevertableRemoveOnDisk( - uint16_t disk, - const document::BucketId& bucketId, - const DocumentId& docId, - Timestamp timestamp) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - spi::RemoveResult result = - getPersistenceProvider().remove( - makeSpiBucket(bucketId, spi::PartitionId(disk)), - spi::Timestamp(timestamp.getTime()), - docId, context); - - return result.wasFound(); -} - -spi::GetResult -MemFileTestUtils::doGetOnDisk( - uint16_t disk, - const document::BucketId& bucketId, - const document::DocumentId& docId, - const document::FieldSet& fields) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - return getPersistenceProvider().get( - makeSpiBucket(bucketId, spi::PartitionId(disk)), - fields, docId, context); -} - -document::DocumentUpdate::SP -MemFileTestUtils::createBodyUpdate( - const document::DocumentId& docId, - const document::FieldValue& updateValue) -{ - const DocumentType* - docType(getTypeRepo()->getDocumentType("testdoctype1")); - document::DocumentUpdate::SP update( - new document::DocumentUpdate(*docType, docId)); - std::shared_ptr assignUpdate( - new document::AssignValueUpdate(updateValue)); - document::FieldUpdate fieldUpdate(docType->getField("content")); - fieldUpdate.addUpdate(*assignUpdate); - update->addUpdate(fieldUpdate); - return update; -} - -document::DocumentUpdate::SP -MemFileTestUtils::createHeaderUpdate( - const document::DocumentId& docId, - const document::FieldValue& updateValue) -{ - const DocumentType* - docType(getTypeRepo()->getDocumentType("testdoctype1")); - document::DocumentUpdate::SP update( - new document::DocumentUpdate(*docType, docId)); - std::shared_ptr assignUpdate( - new document::AssignValueUpdate(updateValue)); - document::FieldUpdate fieldUpdate(docType->getField("headerval")); - fieldUpdate.addUpdate(*assignUpdate); - update->addUpdate(fieldUpdate); - return update; -} - -void -MemFileTestUtils::doPut(const document::Document::SP& doc, - Timestamp time, - uint16_t disk, - uint16_t usedBits) -{ - document::BucketId bucket( - getBucketIdFactory().getBucketId(doc->getId())); - bucket.setUsedBits(usedBits); - doPut(doc, bucket, time, disk); -} - -void -MemFileTestUtils::doPut(const document::Document::SP& doc, - document::BucketId bid, - Timestamp time, - uint16_t disk) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - getPersistenceProvider().put(makeSpiBucket(bid, spi::PartitionId(disk)), - spi::Timestamp(time.getTime()), doc, context); -} - -spi::UpdateResult -MemFileTestUtils::doUpdate(document::BucketId bid, - const document::DocumentUpdate::SP& update, - Timestamp time, - uint16_t disk) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - return getPersistenceProvider().update( - makeSpiBucket(bid, spi::PartitionId(disk)), - spi::Timestamp(time.getTime()), update, context); -} - -void -MemFileTestUtils::doRemove(const document::DocumentId& id, Timestamp time, - uint16_t disk, bool unrevertableRemove, - uint16_t usedBits) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - document::BucketId bucket(getBucketIdFactory().getBucketId(id)); - bucket.setUsedBits(usedBits); - - if (unrevertableRemove) { - getPersistenceProvider().remove( - makeSpiBucket(bucket, spi::PartitionId(disk)), - spi::Timestamp(time.getTime()), - id, context); - } else { - spi::RemoveResult result = getPersistenceProvider().removeIfFound( - makeSpiBucket(bucket, spi::PartitionId(disk)), - spi::Timestamp(time.getTime()), - id, context); - - if (!result.wasFound()) { - throw vespalib::IllegalStateException( - "Attempted to remove non-existing doc " + id.toString(), - VESPA_STRLOC); - } - } -} - -void -MemFileTestUtils::copyHeader(document::Document& dest, - const document::Document& src) -{ - // FIXME(vekterli): temporary solution while we don't have - // fieldset pruning functionality in Document. - //dest.setHeaderPtr(src.getHeaderPtr()); - vespalib::nbostream originalBodyStream; - dest.serializeBody(originalBodyStream); - - vespalib::nbostream headerStream; - src.serializeHeader(headerStream); - document::ByteBuffer hbuf(headerStream.peek(), headerStream.size()); - dest.deserializeHeader(*getTypeRepo(), hbuf); - // deserializeHeader clears fields struct, so have to re-set body - document::ByteBuffer bbuf(originalBodyStream.peek(), - originalBodyStream.size()); - dest.deserializeBody(*getTypeRepo(), bbuf); -} - -void -MemFileTestUtils::copyBody(document::Document& dest, - const document::Document& src) -{ - // FIXME(vekterli): temporary solution while we don't have - // fieldset pruning functionality in Document. - //dest.setBodyPtr(src.getBodyPtr()); - vespalib::nbostream stream; - src.serializeBody(stream); - document::ByteBuffer buf(stream.peek(), stream.size()); - dest.deserializeBody(*getTypeRepo(), buf); -} - -void -MemFileTestUtils::clearBody(document::Document& doc) -{ - // FIXME(vekterli): temporary solution while we don't have - // fieldset pruning functionality in Document. - //doc->getBody().clear(); - vespalib::nbostream stream; - doc.serializeHeader(stream); - doc.deserialize(*getTypeRepo(), stream); -} - -void -MemFileTestUtils::createTestBucket(const document::BucketId& bucket, - uint16_t disk) -{ - - uint32_t opsPerType = 2; - uint32_t numberOfLocations = 2; - uint32_t minDocSize = 0; - uint32_t maxDocSize = 128; - - for (uint32_t useHeaderOnly = 0; useHeaderOnly < 2; ++useHeaderOnly) { - bool headerOnly = (useHeaderOnly == 1); - for (uint32_t optype=0; optype < 4; ++optype) { - for (uint32_t i=0; isetValue(doc2->getField("content"), - document::StringFieldValue("overwritten")); - doPut(doc2, Timestamp(seed + 500), - disk, bucket.getUsedBits()); - } else if (optype == 2) { // Removed - doRemove(doc->getId(), Timestamp(seed + 500), disk, false, - bucket.getUsedBits()); - } else if (optype == 3) { // Unrevertable removed - doRemove(doc->getId(), Timestamp(seed), disk, true, - bucket.getUsedBits()); - } - } - } - } - flush(bucket, disk); -} - -void -MemFileTestUtils::simulateIoErrorsForSubsequentlyOpenedFiles( - const IoErrors& errs) -{ - std::unique_ptr factory( - new SimulatedFailureLazyFile::Factory); - factory->setWriteOpsBeforeFailure(errs._afterWrites); - factory->setReadOpsBeforeFailure(errs._afterReads); - env()._lazyFileFactory = std::move(factory); -} - -void -MemFileTestUtils::unSimulateIoErrorsForSubsequentlyOpenedFiles() -{ - env()._lazyFileFactory = std::unique_ptr( - new DefaultLazyFileFactory(0)); -} - -std::string -MemFileTestUtils::stringifyFields(const document::Document& doc) const -{ - using namespace document; - std::vector output; - const StructFieldValue& fields(doc.getFields()); - for (StructFieldValue::const_iterator - it(fields.begin()), e(fields.end()); - it != e; ++it) - { - std::ostringstream ss; - const Field& f(it.field()); - ss << f.getName() << ": "; - FieldValue::UP val(fields.getValue(f)); - if (val.get()) { - ss << val->toString(); - } else { - ss << "(null)"; - } - output.push_back(ss.str()); - } - std::ostringstream ret; - std::sort(output.begin(), output.end()); - std::copy(output.begin(), output.end(), - std::ostream_iterator(ret, "\n")); - return ret.str(); -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.h b/memfilepersistence/src/tests/spi/memfiletestutils.h deleted file mode 100644 index 657b116b6e5..00000000000 --- a/memfilepersistence/src/tests/spi/memfiletestutils.h +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::memfile::MemFileTestUtils - * \ingroup memfile - * - * \brief Utilities for unit tests of the MemFile layer. - * - * The memfile layer typically needs a MemFileEnvironment object that must be - * set up. This class creates such an object to be used by unit tests. Other - * utilities useful for only MemFile testing can be added here too. - */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace storage { -namespace memfile { - -struct FakeClock : public framework::Clock { -public: - typedef std::unique_ptr UP; - - framework::MicroSecTime _absoluteTime; - - FakeClock() {} - - virtual void addSecondsToTime(uint32_t nr) { - _absoluteTime += framework::MicroSecTime(nr * uint64_t(1000000)); - } - - framework::MicroSecTime getTimeInMicros() const override { - return _absoluteTime; - } - framework::MilliSecTime getTimeInMillis() const override { - return getTimeInMicros().getMillis(); - } - framework::SecondTime getTimeInSeconds() const override { - return getTimeInMicros().getSeconds(); - } - framework::MonotonicTimePoint getMonotonicTime() const override { - return framework::MonotonicTimePoint(std::chrono::microseconds( - getTimeInMicros().getTime())); - } -}; - -struct MemFileTestEnvironment { - MemFileTestEnvironment(uint32_t numDisks, - framework::ComponentRegister& reg, - const document::DocumentTypeRepo& repo); - - vdstestlib::DirConfig _config; - MemFilePersistenceProvider _provider; -}; - -class MemFileTestUtils : public Types, public document::TestDocMan, public CppUnit::TestFixture { -private: - // This variables are kept in test class. Instances that needs to be - // unique per test needs to be setup in setupDisks and cleared in - // tearDown - document::BucketIdFactory _bucketIdFactory; - framework::defaultimplementation::ComponentRegisterImpl::UP _componentRegister; - FakeClock::UP _clock; - std::unique_ptr _env; - -public: - MemFileTestUtils(); - virtual ~MemFileTestUtils(); - - void setupDisks(uint32_t disks); - - void tearDown() override{ - _env.reset(); - _componentRegister.reset(); - _clock.reset(); - } - - std::string getMemFileStatus(const document::BucketId& id, uint32_t disk = 0); - - std::string getModifiedBuckets(); - - /** - Flushes all cached data to disk and updates the bucket database accordingly. - */ - void flush(); - - FakeClock& getFakeClock() { return *_clock; } - - spi::Result flush(const document::BucketId& id, uint16_t disk = 0); - - MemFilePersistenceProvider& getPersistenceProvider(); - - MemFilePtr getMemFile(const document::BucketId& id, uint16_t disk = 0); - - Environment& env(); - - MemFilePersistenceThreadMetrics& getMetrics(); - - MemFileTestEnvironment& getEnv() { return *_env; } - - /** - Performs a put to the given disk. - Returns the document that was inserted. - */ - document::Document::SP doPutOnDisk( - uint16_t disk, - uint32_t location, - Timestamp timestamp, - uint32_t minSize = 0, - uint32_t maxSize = 128); - - document::Document::SP doPut( - uint32_t location, - Timestamp timestamp, - uint32_t minSize = 0, - uint32_t maxSize = 128) - { return doPutOnDisk(0, location, timestamp, minSize, maxSize); } - - /** - Performs a remove to the given disk. - Returns the new doccount if document was removed, or -1 if not found. - */ - bool doRemoveOnDisk( - uint16_t disk, - const document::BucketId& bid, - const document::DocumentId& id, - Timestamp timestamp, - OperationHandler::RemoveType persistRemove); - - bool doRemove( - const document::BucketId& bid, - const document::DocumentId& id, - Timestamp timestamp, - OperationHandler::RemoveType persistRemove) { - return doRemoveOnDisk(0, bid, id, timestamp, persistRemove); - } - - bool doUnrevertableRemoveOnDisk(uint16_t disk, - const document::BucketId& bid, - const DocumentId& id, - Timestamp timestamp); - - bool doUnrevertableRemove(const document::BucketId& bid, - const DocumentId& id, - Timestamp timestamp) - { - return doUnrevertableRemoveOnDisk(0, bid, id, timestamp); - } - - virtual const document::BucketIdFactory& getBucketIdFactory() const - { return _bucketIdFactory; } - - document::BucketIdFactory& getBucketIdFactory() - { return _bucketIdFactory; } - - /** - * Do a remove toward storage set up in test environment. - * - * @id Document to remove. - * @disk If set, use this disk, otherwise lookup in bucket db. - * @unrevertableRemove If set, instead of adding put, turn put to remove. - * @usedBits Generate bucket to use from docid using this amount of bits. - */ - void doRemove(const DocumentId& id, Timestamp, uint16_t disk, - bool unrevertableRemove = false, uint16_t usedBits = 16); - - spi::GetResult doGetOnDisk( - uint16_t disk, - const document::BucketId& bucketId, - const document::DocumentId& docId, - const document::FieldSet& fields); - - spi::GetResult doGet( - const document::BucketId& bucketId, - const document::DocumentId& docId, - const document::FieldSet& fields) - { return doGetOnDisk(0, bucketId, docId, fields); } - - document::DocumentUpdate::SP createBodyUpdate( - const document::DocumentId& id, - const document::FieldValue& updateValue); - - document::DocumentUpdate::SP createHeaderUpdate( - const document::DocumentId& id, - const document::FieldValue& updateValue); - - virtual const std::shared_ptr getTypeRepo() const - { return document::TestDocMan::getTypeRepoSP(); } - - /** - * Do a put toward storage set up in test environment. - * - * @doc Document to put. Use TestDocMan to generate easily. - * @disk If set, use this disk, otherwise lookup in bucket db. - * @usedBits Generate bucket to use from docid using this amount of bits. - */ - void doPut(const Document::SP& doc, Timestamp, - uint16_t disk, uint16_t usedBits = 16); - - void doPut(const document::Document::SP& doc, - document::BucketId bid, - Timestamp time, - uint16_t disk = 0); - - spi::UpdateResult doUpdate(document::BucketId bid, - const document::DocumentUpdate::SP& update, - Timestamp time, - uint16_t disk = 0); - - /** - * Create a test bucket with various content representing most states a - * bucket can represent. (Such that tests have a nice test bucket to use - * that require operations to handle all the various bucket contents. - * - * @disk If set, use this disk, otherwise lookup in bucket db. - */ - void createTestBucket(const BucketId&, uint16_t disk = 0xffff); - - /** - * In-place modify doc so that it has no more body fields. - */ - void clearBody(document::Document& doc); - - /** - * Copy all header data from src into dest, replacing any - * header fields it may already have there. NOTE: this will - * also overwrite document ID, type etc! - */ - void copyHeader(document::Document& dest, - const document::Document& src); - - /** - * Copy all body data from src into dest, replacing any - * body fields it may already have there. - */ - void copyBody(document::Document& dest, - const document::Document& src); - - std::string stringifyFields(const Document& doc) const; - - struct IoErrors { - int _afterReads; - int _afterWrites; - - IoErrors() - : _afterReads(0), - _afterWrites(0) - { - } - - IoErrors& afterReads(int n) { - _afterReads = n; - return *this; - } - - IoErrors& afterWrites(int n) { - _afterWrites = n; - return *this; - } - }; - - /** - * Replaces internal LazyFile factory so that it produces LazyFile - * implementations that trigger I/O exceptions on read/write. Optionally, - * can supply a parameter setting explicit bounds on how many operations - * are allowed on a file before trigging exceptions from there on out. A - * bound of -1 in practice means "don't fail ever" while 0 means "fail the - * next op of that type". - */ - void simulateIoErrorsForSubsequentlyOpenedFiles( - const IoErrors& errs = IoErrors()); - - /** - * Replace internal LazyFile factory with the default, non-failing impl. - */ - void unSimulateIoErrorsForSubsequentlyOpenedFiles(); -}; - -class SingleDiskMemFileTestUtils : public MemFileTestUtils -{ -public: - void setUp() override { - setupDisks(1); - } -}; - -} // memfile -} // storage - diff --git a/memfilepersistence/src/tests/spi/memfilev1serializertest.cpp b/memfilepersistence/src/tests/spi/memfilev1serializertest.cpp deleted file mode 100644 index 9eb2ca00f60..00000000000 --- a/memfilepersistence/src/tests/spi/memfilev1serializertest.cpp +++ /dev/null @@ -1,1100 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include -#include -#include - -namespace storage { -namespace memfile { - -struct MemFileV1SerializerTest : public SingleDiskMemFileTestUtils -{ - void tearDown() override; - void setUpPartialWriteEnvironment(); - void resetConfig(uint32_t minimumFileSize, uint32_t minimumFileHeaderBlockSize); - void doTestPartialWriteRemove(bool readAll); - void doTestPartialWriteUpdate(bool readAll); - - void testWriteReadSingleDoc(); - void testWriteReadPartial(); - void testWriteReadPartialRemoved(); - void testPartialWritePutHeaderOnly(); - void testPartialWritePut(); - void testPartialWriteRemoveCached(); - void testPartialWriteRemoveNotCached(); - void testPartialWriteUpdateCached(); - void testPartialWriteUpdateNotCached(); - void testPartialWriteTooMuchFreeSpace(); - void testPartialWriteNotEnoughFreeSpace(); - void testWriteReadSingleRemovedDoc(); - void testLocationDiskIoPlannerSimple(); - void testLocationDiskIoPlannerMergeReads(); - void testLocationDiskIoPlannerAlignReads(); - void testLocationDiskIoPlannerOneDocument(); - void testSeparateReadsForHeaderAndBody(); - void testLocationsRemappedConsistently(); - void testHeaderBufferTooSmall(); - - /*std::unique_ptr createMemFile(FileSpecification& file, - bool callLoadFile) - { - return std::unique_ptr(new MemFile(file, env(), callLoadFile)); - }*/ - - CPPUNIT_TEST_SUITE(MemFileV1SerializerTest); - CPPUNIT_TEST(testWriteReadSingleDoc); - CPPUNIT_TEST(testWriteReadPartial); - CPPUNIT_TEST(testWriteReadPartialRemoved); - CPPUNIT_TEST(testWriteReadSingleRemovedDoc); - CPPUNIT_TEST(testPartialWritePutHeaderOnly); - CPPUNIT_TEST(testPartialWritePut); - CPPUNIT_TEST(testPartialWriteRemoveCached); - CPPUNIT_TEST(testPartialWriteRemoveNotCached); - CPPUNIT_TEST(testPartialWriteUpdateCached); - CPPUNIT_TEST(testPartialWriteUpdateNotCached); - CPPUNIT_TEST(testLocationDiskIoPlannerSimple); - CPPUNIT_TEST(testLocationDiskIoPlannerMergeReads); - CPPUNIT_TEST(testLocationDiskIoPlannerAlignReads); - CPPUNIT_TEST(testLocationDiskIoPlannerOneDocument); - CPPUNIT_TEST(testSeparateReadsForHeaderAndBody); - CPPUNIT_TEST(testPartialWriteTooMuchFreeSpace); - CPPUNIT_TEST(testPartialWriteNotEnoughFreeSpace); - CPPUNIT_TEST(testLocationsRemappedConsistently); - CPPUNIT_TEST(testHeaderBufferTooSmall); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemFileV1SerializerTest); - -namespace { - -const vespalib::LazyFile& -getFileHandle(const MemFile& mf1) -{ - return static_cast( - mf1.getMemFileIO()).getFileHandle(); -} - -const LoggingLazyFile& -getLoggerFile(const MemFile& file) -{ - return static_cast(getFileHandle(file)); -} - -bool isContentEqual(MemFile& mf1, MemFile& mf2, - bool requireEqualContentCached, std::ostream& error) -{ - MemFile::const_iterator it1( - mf1.begin(Types::ITERATE_GID_UNIQUE | Types::ITERATE_REMOVED)); - MemFile::const_iterator it2( - mf2.begin(Types::ITERATE_GID_UNIQUE | Types::ITERATE_REMOVED)); - while (true) { - if (it1 == mf1.end() && it2 == mf2.end()) { - return true; - } - if (it1 == mf1.end() || it2 == mf2.end()) { - error << "Different amount of GID unique slots"; - return false; - } - if (it1->getTimestamp() != it2->getTimestamp()) { - error << "Different timestamps"; - return false; - } - if (it1->getGlobalId() != it2->getGlobalId()) { - error << "Different gids"; - return false; - } - if (it1->getPersistedFlags() != it2->getPersistedFlags()) { - error << "Different persisted flags"; - return false; - } - if (requireEqualContentCached) { - if (mf1.partAvailable(*it1, Types::BODY) - ^ mf2.partAvailable(*it2, Types::BODY) - || mf1.partAvailable(*it1, Types::HEADER) - ^ mf2.partAvailable(*it2, Types::HEADER)) - { - error << "Difference in cached content: "; - return false; - } - } - - if (mf1.partAvailable(*it1, Types::HEADER) && - mf2.partAvailable(*it2, Types::HEADER)) - { - document::Document::UP doc1 = mf1.getDocument(*it1, Types::ALL); - document::Document::UP doc2 = mf2.getDocument(*it2, Types::ALL); - - CPPUNIT_ASSERT(doc1.get()); - CPPUNIT_ASSERT(doc2.get()); - - if (*doc1 != *doc2) { - error << "Documents different: Expected:\n" - << doc1->toString(true) << "\nActual:\n" - << doc2->toString(true) << "\n"; - return false; - } - } - ++it1; - ++it2; - } -} - -bool -validateMemFileStructure(const MemFile& mf, std::ostream& error) -{ - const SimpleMemFileIOBuffer& ioBuf( - dynamic_cast(mf.getMemFileIO())); - const FileInfo& fileInfo(ioBuf.getFileInfo()); - if (fileInfo.getFileSize() % 512) { - error << "File size is not a multiple of 512 bytes"; - return false; - } - if (fileInfo.getBlockIndex(Types::BODY) % 512) { - error << "Body start index is not a multiple of 512 bytes"; - return false; - } - if (fileInfo.getBlockSize(Types::BODY) % 512) { - error << "Body size is not a multiple of 512 bytes"; - return false; - } - return true; -} - -} - -void -MemFileV1SerializerTest::tearDown() { - //_memFile.reset(); -} - -/** - * Adjust minimum slotfile size values to avoid rewriting file - * when we want to get a partial write - */ -void -MemFileV1SerializerTest::setUpPartialWriteEnvironment() -{ - resetConfig(4096, 2048); -} - -void -MemFileV1SerializerTest::resetConfig(uint32_t minimumFileSize, - uint32_t minimumFileHeaderBlockSize) -{ - using MemFileConfig = vespa::config::storage::StorMemfilepersistenceConfig; - using MemFileConfigBuilder - = vespa::config::storage::StorMemfilepersistenceConfigBuilder; - - MemFileConfigBuilder persistenceConfig( - *env().acquireConfigReadLock().memFilePersistenceConfig()); - persistenceConfig.minimumFileHeaderBlockSize = minimumFileHeaderBlockSize; - persistenceConfig.minimumFileSize = minimumFileSize; - auto newCfg = std::unique_ptr( - new MemFileConfig(persistenceConfig)); - env().acquireConfigWriteLock().setMemFilePersistenceConfig( - std::move(newCfg)); -} - -struct DummyMemFileIOInterface : MemFileIOInterface { - Document::UP getDocumentHeader(const document::DocumentTypeRepo&, - DataLocation) const override - { - return Document::UP(); - } - - document::DocumentId getDocumentId(DataLocation) const override { - return document::DocumentId(""); - } - - void readBody(const document::DocumentTypeRepo&, - DataLocation, - Document&) const override - { - } - DataLocation addDocumentIdOnlyHeader( - const DocumentId&, - const document::DocumentTypeRepo&) override - { - return DataLocation(); - } - DataLocation addHeader(const Document&) override { return DataLocation(); } - DataLocation addBody(const Document&) override { return DataLocation(); } - void clear(DocumentPart) override {} - bool verifyConsistent() const override { return true; } - void move(const FileSpecification&) override {} - DataLocation copyCache(const MemFileIOInterface&, DocumentPart, DataLocation) override { - return DataLocation(); - } - - void close() override {}; - bool isCached(DataLocation, DocumentPart) const override { return false; } - bool isPersisted(DataLocation, DocumentPart) const override { return false; } - uint32_t getSerializedSize(DocumentPart, DataLocation) const override { return 0; } - - void ensureCached(Environment&, DocumentPart, const std::vector&) override {} - - size_t getCachedSize(DocumentPart) const override { return 0; } -}; - -#define VESPA_MEMFILEV1_SETUP_SOURCE \ - system("rm -f testfile.0"); \ - document::Document::SP doc(createRandomDocumentAtLocation(4)); \ - FileSpecification file(document::BucketId(16, 4), env().getDirectory(0), "testfile.0"); \ - MemFile source(file, env()); - -#define VESPA_MEMFILEV1_DIFF(source, target) \ - "\nSource:\n" + source.toString(true) \ - + "\nTarget:\n" + target.toString(true) - -#define VESPA_MEMFILEV1_VALIDATE_STRUCTURE(mfile) \ -{ \ - std::ostringstream validateErr; \ - if (!validateMemFileStructure(mfile, validateErr)) { \ - CPPUNIT_FAIL(validateErr.str()); \ - } \ -} - -#define VESPA_MEMFILEV1_ASSERT_SERIALIZATION(sourceMemFile) \ -env()._memFileMapper.flush(sourceMemFile, env()); \ -VESPA_MEMFILEV1_VALIDATE_STRUCTURE(sourceMemFile) \ -MemFile target(file, env()); \ -VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target) \ -{ \ - target.ensureBodyBlockCached(); \ - target.getBucketInfo(); \ - std::ostringstream diff; \ - if (!isContentEqual(sourceMemFile, target, true, diff)) { \ - std::string msg = "MemFiles not content equal: " + diff.str() \ - + VESPA_MEMFILEV1_DIFF(sourceMemFile, target); \ - CPPUNIT_FAIL(msg); \ - } \ -} - -void -MemFileV1SerializerTest::testWriteReadSingleDoc() -{ - VESPA_MEMFILEV1_SETUP_SOURCE; - source.addPutSlot(*doc, Timestamp(1001)); - std::string foo(VESPA_MEMFILEV1_DIFF(source, source)); - VESPA_MEMFILEV1_ASSERT_SERIALIZATION(source); -} - -void -MemFileV1SerializerTest::testWriteReadPartial() -{ - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - std::map docs; - { - MemFile source(file, env()); - - for (int i = 0; i < 50; ++i) { - Document::SP doc(createRandomDocumentAtLocation(4, i, 1000, 2000)); - source.addPutSlot(*doc, Timestamp(1001 + i)); - docs[Timestamp(1001 + i)] = doc; - } - - env()._memFileMapper.flush(source, env()); - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(source); - } - - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options).maximumReadThroughGap(1024).build()); - env()._lazyFileFactory = std::unique_ptr( - new LoggingLazyFile::Factory()); - - MemFile target(file, env()); - - std::vector timestamps; - - for (int i = 0; i < 50; i+=4) { - timestamps.push_back(Timestamp(1001 + i)); - } - CPPUNIT_ASSERT_EQUAL(size_t(13), timestamps.size()); - - getLoggerFile(target).operations.clear(); - target.ensureDocumentCached(timestamps, false); - // Headers are small enough that they get read in 1 op + 13 body reads - CPPUNIT_ASSERT_EQUAL(14, (int)getLoggerFile(target).operations.size()); - - for (std::size_t i = 0; i < timestamps.size(); ++i) { - const MemSlot* slot = target.getSlotAtTime(timestamps[i]); - CPPUNIT_ASSERT(slot); - CPPUNIT_ASSERT(target.partAvailable(*slot, HEADER)); - CPPUNIT_ASSERT(target.partAvailable(*slot, BODY)); - CPPUNIT_ASSERT_EQUAL(*docs[timestamps[i]], *target.getDocument(*slot, ALL)); - } - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target); -} - -void -MemFileV1SerializerTest::testWriteReadPartialRemoved() -{ - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - MemFile source(file, env()); - - for (int i = 0; i < 50; ++i) { - Document::SP doc(createRandomDocumentAtLocation(4, i, 1000, 2000)); - source.addPutSlot(*doc, Timestamp(1001 + i)); - source.addRemoveSlot(*source.getSlotAtTime(Timestamp(1001 + i)), - Timestamp(2001 + i)); - } - - env()._memFileMapper.flush(source, env()); - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(source); - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options).maximumReadThroughGap(1024).build()); - env()._lazyFileFactory = std::unique_ptr( - new LoggingLazyFile::Factory); - - MemFile target(file, env()); - - std::vector timestamps; - - for (int i = 0; i < 50; i+=4) { - timestamps.push_back(Timestamp(2001 + i)); - } - - getLoggerFile(target).operations.clear(); - target.ensureDocumentCached(timestamps, false); - // All removed; should only read header locations - CPPUNIT_ASSERT_EQUAL(1, (int)getLoggerFile(target).operations.size()); - - for (std::size_t i = 0; i < timestamps.size(); ++i) { - const MemSlot* slot = target.getSlotAtTime(timestamps[i]); - const MemSlot* removedPut( - target.getSlotAtTime(timestamps[i] - Timestamp(1000))); - CPPUNIT_ASSERT(slot); - CPPUNIT_ASSERT(removedPut); - CPPUNIT_ASSERT(target.partAvailable(*slot, HEADER)); - CPPUNIT_ASSERT_EQUAL(removedPut->getLocation(HEADER), - slot->getLocation(HEADER)); - CPPUNIT_ASSERT_EQUAL(DataLocation(0, 0), slot->getLocation(BODY)); - } - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target); -} - -void MemFileV1SerializerTest::testWriteReadSingleRemovedDoc() -{ - VESPA_MEMFILEV1_SETUP_SOURCE; - source.addPutSlot(*doc, Timestamp(1001)); - source.addRemoveSlot( - *source.getSlotAtTime(Timestamp(1001)), Timestamp(2001)); - VESPA_MEMFILEV1_ASSERT_SERIALIZATION(source); -} - -/** - * Write a single put with no body to the memfile and ensure it is - * persisted properly without a body block - */ -void -MemFileV1SerializerTest::testPartialWritePutHeaderOnly() -{ - setUpPartialWriteEnvironment(); - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - document::Document::SP doc(createRandomDocumentAtLocation(4)); - { - MemFile source(file, env()); - source.addPutSlot(*doc, Timestamp(1001)); - env()._memFileMapper.flush(source, env()); - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(source); - } - { - // Have to put a second time since the first one will always - // rewrite the entire file - MemFile target(file, env()); - Document::SP doc2(createRandomDocumentAtLocation(4)); - clearBody(*doc2); - target.addPutSlot(*doc2, Timestamp(1003)); - env()._memFileMapper.flush(target, env()); - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target); - } - { - MemFile target(file, env()); - target.ensureBodyBlockCached(); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getSlotCount()); - - const MemSlot& slot = *target.getSlotAtTime(Timestamp(1003)); - CPPUNIT_ASSERT(slot.getLocation(HEADER)._pos > 0); - CPPUNIT_ASSERT(slot.getLocation(HEADER)._size > 0); - CPPUNIT_ASSERT_EQUAL( - DataLocation(0, 0), slot.getLocation(BODY)); - VESPA_MEMFILEV1_VALIDATE_STRUCTURE(target); - } -} - - - - -void -MemFileV1SerializerTest::testLocationDiskIoPlannerSimple() -{ - std::vector slots; - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1001), - DataLocation(0, 1024), - DataLocation(4096, 512), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1003), - DataLocation(1024, 1024), - DataLocation(8192, 512), 0, 0)); - } - - std::vector headers; - std::vector bodies; - headers.push_back(slots[0].getLocation(HEADER)); - bodies.push_back(slots[0].getLocation(BODY)); - - DummyMemFileIOInterface dummyIo; - { - LocationDiskIoPlanner planner(dummyIo, HEADER, headers, 100, 0); - - CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size()); - CPPUNIT_ASSERT_EQUAL( - DataLocation(0, 1024), - planner.getIoOperations()[0]); - } - { - LocationDiskIoPlanner planner(dummyIo, BODY, bodies, 100, 4096); - - CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size()); - CPPUNIT_ASSERT_EQUAL( - DataLocation(8192, 512), // + block index - planner.getIoOperations()[0]); - } -} - -void -MemFileV1SerializerTest::testLocationDiskIoPlannerMergeReads() -{ - std::vector slots; - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1001), - DataLocation(0, 1024), - DataLocation(5120, 512), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1002), - DataLocation(2048, 1024), - DataLocation(7168, 512), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1003), - DataLocation(1024, 1024), - DataLocation(9216, 512), 0, 0)); - } - - std::vector headers; - std::vector bodies; - for (int i = 0; i < 2; ++i) { - headers.push_back(slots[i].getLocation(HEADER)); - bodies.push_back(slots[i].getLocation(BODY)); - } - - DummyMemFileIOInterface dummyIo; - { - LocationDiskIoPlanner planner(dummyIo, HEADER, headers, 1025, 0); - - CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size()); - CPPUNIT_ASSERT_EQUAL( - DataLocation(0, 3072), - planner.getIoOperations()[0]); - } - - { - LocationDiskIoPlanner planner(dummyIo, BODY, bodies, 1025, 0); - - CPPUNIT_ASSERT_EQUAL(2, (int)planner.getIoOperations().size()); - CPPUNIT_ASSERT_EQUAL( - DataLocation(5120, 512), - planner.getIoOperations()[0]); - CPPUNIT_ASSERT_EQUAL( - DataLocation(7168, 512), - planner.getIoOperations()[1]); - } -} - -void -MemFileV1SerializerTest::testLocationDiskIoPlannerOneDocument() -{ - std::vector slots; - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1001), - DataLocation(0, 1024), - DataLocation(5120, 512), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1002), - DataLocation(2048, 1024), - DataLocation(7168, 512), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1003), - DataLocation(1024, 1024), - DataLocation(9216, 512), 0, 0)); - } - - std::vector headers; - std::vector bodies; - headers.push_back(slots[1].getLocation(HEADER)); - bodies.push_back(slots[1].getLocation(BODY)); - - DummyMemFileIOInterface dummyIo; - { - LocationDiskIoPlanner planner(dummyIo, HEADER, headers, 1000, 0); - CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size()); - CPPUNIT_ASSERT_EQUAL( - DataLocation(2048, 1024), - planner.getIoOperations()[0]); - } - - { - LocationDiskIoPlanner planner(dummyIo, BODY, bodies, 1000, 0); - CPPUNIT_ASSERT_EQUAL(1, (int)planner.getIoOperations().size()); - CPPUNIT_ASSERT_EQUAL( - DataLocation(7168, 512), - planner.getIoOperations()[0]); - } -} - -void -MemFileV1SerializerTest::testLocationDiskIoPlannerAlignReads() -{ - std::vector slots; - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1001), - DataLocation(7, 100), - DataLocation(5000, 500), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1002), - DataLocation(2000, 100), - DataLocation(7000, 500), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1003), - DataLocation(110, 200), - DataLocation(9000, 500), 0, 0)); - } - - { - Document::SP doc(createRandomDocumentAtLocation(4)); - slots.push_back( - MemSlot( - doc->getId().getGlobalId(), - Timestamp(1004), - DataLocation(3000, 100), - DataLocation(11000, 500), 0, 0)); - } - - std::vector headers; - std::vector bodies; - for (int i = 0; i < 2; ++i) { - headers.push_back(slots[i].getLocation(HEADER)); - bodies.push_back(slots[i].getLocation(BODY)); - } - - DummyMemFileIOInterface dummyIo; - { - LocationDiskIoPlanner planner(dummyIo, HEADER, headers, 512, 0); - std::vector expected; - expected.push_back(DataLocation(0, 512)); - expected.push_back(DataLocation(1536, 1024)); - - CPPUNIT_ASSERT_EQUAL(expected, planner.getIoOperations()); - } - { - LocationDiskIoPlanner planner(dummyIo, BODY, bodies, 512, 0); - std::vector expected; - expected.push_back(DataLocation(4608, 1024)); - expected.push_back(DataLocation(6656, 1024)); - - CPPUNIT_ASSERT_EQUAL(expected, planner.getIoOperations()); - } -} - -// TODO(vekterli): add read planner test with a location cached - -void -MemFileV1SerializerTest::testSeparateReadsForHeaderAndBody() -{ - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - Document::SP doc(createRandomDocumentAtLocation(4, 0, 1000, 2000)); - { - MemFile source(file, env()); - source.addPutSlot(*doc, Timestamp(1001)); - - env()._memFileMapper.flush(source, env()); - } - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options) - .maximumReadThroughGap(1024*1024*100) - .build()); - env()._lazyFileFactory = std::unique_ptr( - new LoggingLazyFile::Factory()); - - MemFile target(file, env()); - - std::vector timestamps; - timestamps.push_back(Timestamp(1001)); - - getLoggerFile(target).operations.clear(); - target.ensureDocumentCached(timestamps, false); - - CPPUNIT_ASSERT_EQUAL(2, (int)getLoggerFile(target).operations.size()); - const MemSlot* slot = target.getSlotAtTime(Timestamp(1001)); - CPPUNIT_ASSERT(slot); - CPPUNIT_ASSERT(target.partAvailable(*slot, HEADER)); - CPPUNIT_ASSERT(target.partAvailable(*slot, BODY)); - CPPUNIT_ASSERT_EQUAL(*doc, *target.getDocument(*slot, ALL)); - - CPPUNIT_ASSERT(getMetrics().serialization.headerReadSize.getLast() > 0); - CPPUNIT_ASSERT(getMetrics().serialization.bodyReadSize.getLast() > 0); -} - -/** - * Write a single put with body to the memfile and ensure it is - * persisted properly with both header and body blocks - */ -void -MemFileV1SerializerTest::testPartialWritePut() -{ - setUpPartialWriteEnvironment(); - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - Document::SP doc(createRandomDocumentAtLocation(4)); - { - MemFile source(file, env()); - source.addPutSlot(*doc, Timestamp(1001)); - - env()._memFileMapper.flush(source, env()); - } - - { - // Have to put a second time since the first one will always - // rewrite the entire file - MemFile target(file, env()); - Document::SP doc2(createRandomDocumentAtLocation(4)); - target.addPutSlot(*doc2, Timestamp(1003)); - env()._memFileMapper.flush(target, env()); - } - { - MemFile target(file, env()); - target.ensureBodyBlockCached(); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getSlotCount()); - - const MemSlot& slot = *target.getSlotAtTime(Timestamp(1003)); - CPPUNIT_ASSERT(slot.getLocation(HEADER)._pos > 0); - CPPUNIT_ASSERT(slot.getLocation(HEADER)._size > 0); - - CPPUNIT_ASSERT(slot.getLocation(BODY)._size > 0); - CPPUNIT_ASSERT(slot.getLocation(BODY)._pos > 0); - } -} - -void -MemFileV1SerializerTest::doTestPartialWriteRemove(bool readAll) -{ - setUpPartialWriteEnvironment(); - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - Document::SP doc(createRandomDocumentAtLocation(4)); - { - MemFile source(file, env()); - source.addPutSlot(*doc, Timestamp(1001)); - env()._memFileMapper.flush(source, env()); - } - { - MemFile target(file, env()); - // Only populate cache before removing if explicitly told so - if (readAll) { - target.ensureBodyBlockCached(); - } - CPPUNIT_ASSERT_EQUAL(uint32_t(1), target.getSlotCount()); - target.addRemoveSlot(target[0], Timestamp(1003)); - - env()._memFileMapper.flush(target, env()); - } - { - MemFile target(file, env()); - target.ensureBodyBlockCached(); - - CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getSlotCount()); - - const MemSlot& originalSlot = target[0]; - const MemSlot& removeSlot = target[1]; - CPPUNIT_ASSERT(originalSlot.getLocation(HEADER)._size > 0); - CPPUNIT_ASSERT(originalSlot.getLocation(BODY)._size > 0); - CPPUNIT_ASSERT_EQUAL( - originalSlot.getLocation(HEADER), - removeSlot.getLocation(HEADER)); - CPPUNIT_ASSERT_EQUAL( - DataLocation(0, 0), removeSlot.getLocation(BODY)); - } -} - -/** - * Ensure that removes get the same header location as the Put - * they're removing, and that they get a zero body location - */ -void -MemFileV1SerializerTest::testPartialWriteRemoveCached() -{ - doTestPartialWriteRemove(true); -} - -void -MemFileV1SerializerTest::testPartialWriteRemoveNotCached() -{ - doTestPartialWriteRemove(false); -} - -void -MemFileV1SerializerTest::doTestPartialWriteUpdate(bool readAll) -{ - setUpPartialWriteEnvironment(); - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - Document::SP doc(createRandomDocumentAtLocation(4)); - { - MemFile source(file, env()); - source.addPutSlot(*doc, Timestamp(1001)); - env()._memFileMapper.flush(source, env()); - } - - Document::SP doc2; - { - MemFile target(file, env()); - if (readAll) { - target.ensureBodyBlockCached(); - } - - doc2.reset(new Document(*doc->getDataType(), doc->getId())); - clearBody(*doc2); - doc2->setValue(doc->getField("hstringval"), - document::StringFieldValue("Some updated content")); - - target.addUpdateSlot(*doc2, *target.getSlotAtTime(Timestamp(1001)), - Timestamp(1003)); - env()._memFileMapper.flush(target, env()); - } - - { - MemFile target(file, env()); - CPPUNIT_ASSERT_EQUAL(uint32_t(2), target.getSlotCount()); - const MemSlot& originalSlot = target[0]; - const MemSlot& updateSlot = target[1]; - CPPUNIT_ASSERT(originalSlot.getLocation(HEADER)._size > 0); - CPPUNIT_ASSERT(originalSlot.getLocation(BODY)._size > 0); - CPPUNIT_ASSERT_EQUAL( - originalSlot.getLocation(BODY), - updateSlot.getLocation(BODY)); - CPPUNIT_ASSERT( - updateSlot.getLocation(HEADER) - != originalSlot.getLocation(HEADER)); - - CPPUNIT_ASSERT_EQUAL(*doc, *target.getDocument(target[0], ALL)); - copyHeader(*doc, *doc2); - CPPUNIT_ASSERT_EQUAL(*doc, *target.getDocument(target[1], ALL)); - } -} - -/** - * Ensure that header updates keep the same body block - */ -void -MemFileV1SerializerTest::testPartialWriteUpdateCached() -{ - doTestPartialWriteUpdate(true); -} - -void -MemFileV1SerializerTest::testPartialWriteUpdateNotCached() -{ - doTestPartialWriteUpdate(false); -} - -void -MemFileV1SerializerTest::testPartialWriteTooMuchFreeSpace() -{ - setUpPartialWriteEnvironment(); - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - { - MemFile source(file, env()); - Document::SP doc(createRandomDocumentAtLocation(4)); - source.addPutSlot(*doc, Timestamp(1001)); - env()._memFileMapper.flush(source, env()); - } - int64_t sizeBefore; - // Append filler to slotfile to make it too big for comfort, - // forcing a rewrite to shrink it down - { - vespalib::File slotfile(file.getPath()); - slotfile.open(0); - CPPUNIT_ASSERT(slotfile.isOpen()); - sizeBefore = slotfile.getFileSize(); - slotfile.resize(sizeBefore * 20); // Well over min fill rate of 10% - } - // Write new slot to file; it should now be rewritten with the - // same file size as originally - { - MemFile source(file, env()); - Document::SP doc(createRandomDocumentAtLocation(4)); - source.addPutSlot(*doc, Timestamp(1003)); - env()._memFileMapper.flush(source, env()); - } - { - vespalib::File slotfile(file.getPath()); - slotfile.open(0); - CPPUNIT_ASSERT(slotfile.isOpen()); - CPPUNIT_ASSERT_EQUAL( - sizeBefore, - slotfile.getFileSize()); - } - CPPUNIT_ASSERT_EQUAL(uint64_t(1), getMetrics().serialization - .fullRewritesDueToDownsizingFile.getValue()); - CPPUNIT_ASSERT_EQUAL(uint64_t(0), getMetrics().serialization - .fullRewritesDueToTooSmallFile.getValue()); -} - -void -MemFileV1SerializerTest::testPartialWriteNotEnoughFreeSpace() -{ - setUpPartialWriteEnvironment(); - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - // Write file initially - MemFile source(file, env()); - { - Document::SP doc(createRandomDocumentAtLocation(4)); - source.addPutSlot(*doc, Timestamp(1001)); - env()._memFileMapper.flush(source, env()); - } - - uint32_t minFile = 1024 * 512; - auto memFileCfg = env().acquireConfigReadLock().memFilePersistenceConfig(); - resetConfig(minFile, memFileCfg->minimumFileHeaderBlockSize); - - // Create doc bigger than initial minimum filesize, - // prompting a full rewrite - Document::SP doc( - createRandomDocumentAtLocation(4, 0, 4096, 4096)); - source.addPutSlot(*doc, Timestamp(1003)); - - env()._memFileMapper.flush(source, env()); - - CPPUNIT_ASSERT_EQUAL( - minFile, - uint32_t(getFileHandle(source).getFileSize())); - - CPPUNIT_ASSERT_EQUAL(uint64_t(0), getMetrics().serialization - .fullRewritesDueToDownsizingFile.getValue()); - CPPUNIT_ASSERT_EQUAL(uint64_t(1), getMetrics().serialization - .fullRewritesDueToTooSmallFile.getValue()); - - // Now, ensure we respect minimum file size and don't try to - // "helpfully" rewrite the file again (try to detect full - // file rewrite with help from the fact we don't currently - // check whether or not the file is < the minimum filesize. - // If that changes, so must this) - memFileCfg = env().acquireConfigReadLock().memFilePersistenceConfig(); - resetConfig(2 * minFile, memFileCfg->minimumFileHeaderBlockSize); - - source.addRemoveSlot(*source.getSlotAtTime(Timestamp(1003)), - Timestamp(1005)); - env()._memFileMapper.flush(source, env()); - - CPPUNIT_ASSERT_EQUAL( - minFile, - uint32_t(getFileHandle(source).getFileSize())); - - CPPUNIT_ASSERT_EQUAL(uint64_t(1), getMetrics().serialization - .fullRewritesDueToTooSmallFile.getValue()); -} - -// Test that we don't mess up when remapping locations that -// have already been written during the same operation. That is: -// part A is remapped (P1, S1) -> (P2, S2) -// part B is remapped (P2, S2) -> (P3, S3) -// Obviously, part B should not overwrite the location of part A, -// but this will happen if we don't do the updating in one batch. -void -MemFileV1SerializerTest::testLocationsRemappedConsistently() -{ - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - - std::map docs; - { - MemFile mf(file, env()); - Document::SP tmpDoc( - createRandomDocumentAtLocation(4, 0, 100, 100)); - - // Create docs identical in size but differing only in doc ids - // By keeping same size but inserting with _lower_ timestamps - // for docs that get higher location positions, we ensure that - // when the file is rewritten, the lower timestamp slots will - // get remapped to locations that match existing locations for - // higher timestamp slots. - for (int i = 0; i < 2; ++i) { - std::ostringstream ss; - ss << "doc" << i; - DocumentId id(document::UserDocIdString("userdoc:foo:4:" + ss.str())); - Document::SP doc(new Document(*tmpDoc->getDataType(), id)); - doc->getFields() = tmpDoc->getFields(); - mf.addPutSlot(*doc, Timestamp(1000 - i)); - docs[Timestamp(1000 - i)] = doc; - } - - env()._memFileMapper.flush(mf, env()); - // Dirty the cache for rewrite - { - DocumentId id2(document::UserDocIdString("userdoc:foo:4:doc9")); - Document::UP doc2(new Document(*tmpDoc->getDataType(), id2)); - doc2->getFields() = tmpDoc->getFields(); - mf.addPutSlot(*doc2, Timestamp(2000)); - docs[Timestamp(2000)] = std::move(doc2); - } - - // Force rewrite - auto memFileCfg = env().acquireConfigReadLock() - .memFilePersistenceConfig(); - resetConfig(1024*512, memFileCfg ->minimumFileHeaderBlockSize); - env()._memFileMapper.flush(mf, env()); - } - - MemFile target(file, env()); - target.ensureBodyBlockCached(); - - std::ostringstream err; - if (!env()._memFileMapper.verify(target, env(), err)) { - std::cerr << err.str() << "\n"; - CPPUNIT_FAIL("MemFile verification failed"); - } - - typedef std::map::iterator Iter; - for (Iter it(docs.begin()); it != docs.end(); ++it) { - const MemSlot* slot = target.getSlotAtTime(it->first); - CPPUNIT_ASSERT(slot); - CPPUNIT_ASSERT(target.partAvailable(*slot, HEADER)); - CPPUNIT_ASSERT(target.partAvailable(*slot, BODY)); - CPPUNIT_ASSERT_EQUAL(*it->second, *target.getDocument(*slot, ALL)); - } -} - -/** - * Test that we read in the correct header information when we have to read - * in two passes to get it in its entirety. - */ -void -MemFileV1SerializerTest::testHeaderBufferTooSmall() -{ - system("rm -f testfile.0"); - FileSpecification file(BucketId(16, 4), env().getDirectory(0), "testfile.0"); - FileInfo wantedInfo; - { - MemFile f(file, env()); - // 50*40 bytes of meta list data should be more than sufficient - for (size_t i = 0; i < 50; ++i) { - Document::SP doc(createRandomDocumentAtLocation(4, i)); - f.addPutSlot(*doc, Timestamp(1001 + i)); - env()._memFileMapper.flush(f, env()); - } - SimpleMemFileIOBuffer& io( - dynamic_cast(f.getMemFileIO())); - wantedInfo = io.getFileInfo(); - } - - // Force initial index read to be too small to contain all metadata, - // triggering buffer resize and secondary read. - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options).initialIndexRead(512).build()); - { - MemFile f(file, env()); - CPPUNIT_ASSERT_EQUAL(uint32_t(50), f.getSlotCount()); - // Ensure we've read correct file info - SimpleMemFileIOBuffer& io( - dynamic_cast(f.getMemFileIO())); - const FileInfo& info(io.getFileInfo()); - CPPUNIT_ASSERT_EQUAL(wantedInfo.getFileSize(), info.getFileSize()); - CPPUNIT_ASSERT_EQUAL(wantedInfo.getHeaderBlockStartIndex(), - info.getHeaderBlockStartIndex()); - CPPUNIT_ASSERT_EQUAL(wantedInfo.getBodyBlockStartIndex(), - info.getBodyBlockStartIndex()); - CPPUNIT_ASSERT_EQUAL(wantedInfo.getBlockSize(HEADER), - info.getBlockSize(HEADER)); - CPPUNIT_ASSERT_EQUAL(wantedInfo.getBlockSize(BODY), - info.getBlockSize(BODY)); - } -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp b/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp deleted file mode 100644 index c38842bfeb0..00000000000 --- a/memfilepersistence/src/tests/spi/memfilev1verifiertest.cpp +++ /dev/null @@ -1,496 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include - -namespace storage { -namespace memfile { - -struct MemFileV1VerifierTest : public SingleDiskMemFileTestUtils -{ - void testVerify(); - - void tearDown() override; - - std::unique_ptr createMemFile(FileSpecification& file, - bool callLoadFile) - { - return std::unique_ptr(new MemFile(file, env(), callLoadFile)); - } - - CPPUNIT_TEST_SUITE(MemFileV1VerifierTest); - CPPUNIT_TEST_IGNORED(testVerify); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(MemFileV1VerifierTest); - -namespace { - // A totall uncached memfile with content to use for verify testing - std::unique_ptr _memFile; - - // Clear old content. Create new file. Make sure nothing is cached. - void prepareBucket(SingleDiskMemFileTestUtils& util, - const FileSpecification& file) { - _memFile.reset(); - util.env()._cache.clear(); - vespalib::unlink(file.getPath()); - util.createTestBucket(file.getBucketId(), 0); - util.env()._cache.clear(); - _memFile.reset(new MemFile(file, util.env())); - _memFile->getMemFileIO().close(); - - } - - // Get copy of header of memfile created - Header getHeader() { - assert(_memFile.get()); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - Header result; - file.read(&result, sizeof(Header), 0); - return result; - } - - MetaSlot getSlot(uint32_t index) { - assert(_memFile.get()); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - MetaSlot result; - file.read(&result, sizeof(MetaSlot), - sizeof(Header) + sizeof(MetaSlot) * index); - return result; - } - - void setSlot(uint32_t index, MetaSlot slot, - bool updateFileChecksum = true) - { - (void)updateFileChecksum; - assert(_memFile.get()); - //if (updateFileChecksum) slot.updateFileChecksum(); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - file.write(&slot, sizeof(MetaSlot), - sizeof(Header) + sizeof(MetaSlot) * index); - } - - void setHeader(const Header& header) { - assert(_memFile.get()); - vespalib::LazyFile file(_memFile->getFile().getPath(), 0); - file.write(&header, sizeof(Header), 0); - } - - void verifySlotFile(MemFileV1VerifierTest& util, - const std::string& expectedError, - const std::string& message, - int32_t remainingEntries, - bool includeContent = true, - bool includeHeader = true) - { - assert(_memFile.get()); - FileSpecification file(_memFile->getFile()); - _memFile.reset(); - _memFile = util.createMemFile(file, false); - std::ostringstream before; - try{ - util.env()._memFileMapper.loadFile(*_memFile, util.env(), false); - _memFile->print(before, true, ""); - } catch (vespalib::Exception& e) { - before << "Unknown. Exception during loadFile\n"; - } - std::ostringstream errors; - uint32_t flags = (includeContent ? 0 : Types::DONT_VERIFY_BODY) - | (includeHeader ? 0 : Types::DONT_VERIFY_HEADER); - if (util.env()._memFileMapper.verify( - *_memFile, util.env(), errors, flags)) - { - _memFile->print(std::cerr, true, ""); - std::cerr << errors.str() << "\n"; - CPPUNIT_FAIL("verify() failed to detect: " + message); - } - CPPUNIT_ASSERT_CONTAIN_MESSAGE(message + "\nBefore: " + before.str(), - expectedError, errors.str()); - errors.str(""); - if (util.env()._memFileMapper.repair( - *_memFile, util.env(), errors, flags)) - { - CPPUNIT_FAIL("repair() failed to detect: " + message - + ": " + errors.str()); - } - CPPUNIT_ASSERT_CONTAIN_MESSAGE(message + "\nBefore: " + before.str(), - expectedError, errors.str()); - std::ostringstream remainingErrors; - if (!util.env()._memFileMapper.verify( - *_memFile, util.env(), remainingErrors, flags)) - { - CPPUNIT_FAIL("verify() returns issue after repair of: " - + message + ": " + remainingErrors.str()); - } - CPPUNIT_ASSERT_MESSAGE(remainingErrors.str(), - remainingErrors.str().size() == 0); - if (remainingEntries < 0) { - if (_memFile->fileExists()) { - CPPUNIT_FAIL(message + ": Expected file to not exist anymore"); - } - } else if (dynamic_cast(_memFile->getMemFileIO()) - .getFileHandle().getFileSize() == 0) - { - std::ostringstream ost; - ost << "Expected " << remainingEntries << " to remain in file, " - << "but file does not exist\n"; - CPPUNIT_FAIL(message + ": " + ost.str()); - } else { - if (int64_t(_memFile->getSlotCount()) != remainingEntries) { - std::ostringstream ost; - ost << "Expected " << remainingEntries << " to remain in file, " - << "but found " << _memFile->getSlotCount() << "\n"; - ost << errors.str() << "\n"; - ost << "Before: " << before.str() << "\nAfter: "; - _memFile->print(ost, true, ""); - CPPUNIT_FAIL(message + ": " + ost.str()); - } - } - } -} - -void -MemFileV1VerifierTest::tearDown() -{ - _memFile.reset(0); - SingleDiskMemFileTestUtils::tearDown(); -}; - -void -MemFileV1VerifierTest::testVerify() -{ - BucketId bucket(16, 0xa); - std::unique_ptr file; - createTestBucket(bucket, 0); - - { - MemFilePtr memFilePtr(env()._cache.get(bucket, env(), env().getDirectory())); - file.reset(new FileSpecification(memFilePtr->getFile())); - env()._cache.clear(); - } - { // Ensure buildTestFile builds a valid file - // Initial file should be fine. - MemFile memFile(*file, env()); - std::ostringstream errors; - if (!env()._memFileMapper.verify(memFile, env(), errors)) { - memFile.print(std::cerr, false, ""); - CPPUNIT_FAIL("Slotfile failed verification: " + errors.str()); - } - } - // Header tests - prepareBucket(*this, *file); - Header orgheader(getHeader()); - { // Test wrong version - Header header(orgheader); - header.setVersion(0xc0edbabe); - header.updateChecksum(); - setHeader(header); - verifySlotFile(*this, - "400000000000000a.0 is of wrong version", - "Faulty version", - -1); - } - { // Test meta data list size bigger than file - prepareBucket(*this, *file); - Header header(orgheader); - header.setMetaDataListSize(0xFFFF); - header.updateChecksum(); - setHeader(header); - verifySlotFile(*this, - "indicates file is bigger than it physically is", - "Too big meta data list size", - -1); - } - { // Test header block size bigger than file - prepareBucket(*this, *file); - Header header(orgheader); - header.setHeaderBlockSize(0xFFFF); - header.updateChecksum(); - setHeader(header); - verifySlotFile(*this, - "Header indicates file is bigger than it physically is", - "Too big header block size", - -1); - } - { // Test wrong header crc - prepareBucket(*this, *file); - Header header(orgheader); - header.setMetaDataListSize(4); - setHeader(header); - verifySlotFile(*this, - "Header checksum mismatch", - "Wrong header checksum", - -1); - } - // Meta data tests - prepareBucket(*this, *file); - MetaSlot slot6(getSlot(6)); - { // Test extra removes - currently allowed - MetaSlot slot7(getSlot(7)); - MetaSlot s(slot7); - s.setTimestamp(Timestamp(s._timestamp.getTime() - 1)); - s.updateChecksum(); - setSlot(6, s); - s.setTimestamp(Timestamp(s._timestamp.getTime() + 1)); - s.updateChecksum(); - setSlot(7, s); - std::ostringstream errors; - if (!env()._memFileMapper.verify(*_memFile, env(), errors)) { - _memFile->print(std::cerr, false, ""); - std::cerr << errors.str() << "\n"; - CPPUNIT_FAIL("Supposed to be legal with multiple remove values"); - } - setSlot(7, slot7); - } - { - // Test metadata crc mismatch with "used" flag being accidentally - // flipped. Should not inhibit adding of subsequent slots. - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setUseFlag(false); - setSlot(6, s); - verifySlotFile(*this, - "Slot 6 at timestamp 2001 failed checksum verification", - "Crc failure with use flag", 23, false); - } - { // Test overlapping documents - MetaSlot s(slot6); - // Direct overlapping header - prepareBucket(*this, *file); - s.setHeaderPos(0); - s.setHeaderSize(51); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "overlaps with slot", - "Direct overlapping header", 6, false, false); - // Contained header - // (contained bit not valid header so fails on other error now) - prepareBucket(*this, *file); - s.setHeaderPos(176); - s.setHeaderSize(80); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "not big enough to contain a document id", - "Contained header", 7, false); - // Partly overlapping header - // (contained bit not valid header so fails on other error now) - prepareBucket(*this, *file); - s.setHeaderPos(191); - s.setHeaderSize(35); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "not big enough to contain a document id", - "Partly overlapping header", 7, false); - prepareBucket(*this, *file); - s.setHeaderPos(185); - s.setHeaderSize(33); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "not big enough to contain a document id", - "Partly overlapping header (2)", 7, false); - // Direct overlapping body - prepareBucket(*this, *file); - s = slot6; - s.setBodyPos(0); - s.setBodySize(136); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Multiple slots with different gids use same body position", - "Directly overlapping body", 6, false); - // Contained body - prepareBucket(*this, *file); - s.setBodyPos(10); - s.setBodySize(50); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "overlaps with slot", - "Contained body", 6, false); - CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(1)) == 0); - // Overlapping body - prepareBucket(*this, *file); - s.setBodyPos(160); - s.setBodySize(40); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "overlaps with slot", - "Overlapping body", 5, false); - CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(2)) == 0); - CPPUNIT_ASSERT(_memFile->getSlotAtTime(Timestamp(1501)) == 0); - // Overlapping body, verifying bodies - // (Bad body bit should be removed first, so only one slot needs - // removing) - prepareBucket(*this, *file); - setSlot(6, s); - verifySlotFile(*this, - "Body checksum mismatch", - "Overlapping body(2)", 7, true); - } - { // Test out of bounds - MetaSlot s(slot6); - - // Header out of bounds - prepareBucket(*this, *file); - s.setHeaderPos(500); - s.setHeaderSize(5000); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "goes out of bounds", - "Header out of bounds", 7, false, false); - // Body out of bounds - prepareBucket(*this, *file); - s = slot6; - s.setBodyPos(2400); - s.setBodySize(6000); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "goes out of bounds", - "Body out of bounds", 7, false); - } - { // Test timestamp collision - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setTimestamp(Timestamp(10002)); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "has same timestamp as slot 5", - "Timestamp collision", 6, false); - } - { // Test timestamp out of order - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setTimestamp(Timestamp(38)); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Slot 6 is out of timestamp order", - "Timestamp out of order", 8, false); - } - { // Test metadata crc mismatch - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setTimestamp(Timestamp(40)); - setSlot(6, s); - verifySlotFile(*this, - "Slot 6 at timestamp 40 failed checksum verification", - "Crc failure", 7, false); - } - { // Test used after unused - // This might actually lose documents after the unused entries. - // The memfile will not know about the documents after unused entry. - // If the memfile contains changes and writes metadata back due to this, - // the following entries will be missing. - // (To prevent this repair would have to add metadata entries, but that - // may be problems if repair happens at a time where all header or body - // data in the file needs to be cached.) - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setUseFlag(false); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Slot 7 found after unused entries", - "Used after unused", 6, false); - } - { // Test header blob corrupt - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setHeaderPos(519); - s.setHeaderSize(86); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Header checksum mismatch", - "Corrupt header blob.", 7); - } - { // Test body blob corrupt - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setBodyPos(52); - s.setBodySize(18); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "Body checksum mismatch", - "Corrupt body blob.", 7); - } - { // Test too long name for header chunk - prepareBucket(*this, *file); - MetaSlot s(slot6); - s.setHeaderPos(160); - s.setHeaderSize(33); - s.updateChecksum(); - setSlot(6, s); - verifySlotFile(*this, - "header is not big enough to contain a document", - "Too long name in header.", 7); - } - { // Test wrong file checksum -// Currently disabled. Currently only possible to calculate file checksum from -// memfile now, and memfile object wont be valid. -/* - // First test if we actually have less entries at all.. - prepareBucket(*this, *file); - MetaSlot s(getSlot(7)); - s.setUseFlag(false); - s.updateChecksum(); - setSlot(7, s, false); - s = getSlot(8); - s.setUseFlag(false); - s.updateChecksum(); - setSlot(8, s, false); - verifySlotFile(*this, - "File checksum should have been", - "Wrong file checksum in file.", 7, false); -std::cerr << "U\n"; - // Then test with different timestamp in remaining document - prepareBucket(*this, *file); - s = getSlot(6); - s.setTimestamp(s._timestamp + 1); - s.updateChecksum(); - setSlot(6, s, false); - verifySlotFile(*this, - "File checksum should have been", - "Wrong file checksum in file.", 9, false); -std::cerr << "V\n"; - // Then check with different gid - prepareBucket(*this, *file); - s = getSlot(6); - s._gid = GlobalId("sdfsdfsedsdfsdfsd"); - s.updateChecksum(); - setSlot(6, s, false); - verifySlotFile(*this, - "File checksum should have been", - "Wrong file checksum in file.", 9, false, false); -*/ - } - { // Test that documents not belonging in a bucket is removed -// Currently disabled. Hard to test. Needs total rewrite -/* - prepareBucket(*this, *file); - Blob b(createBlob(43u, "userdoc::0:315", "header", "body")); - _memFile->write(b, 80); - CPPUNIT_ASSERT_EQUAL(4u, _memFile->getBlobCount()); - CPPUNIT_ASSERT(_memFile->read(b)); - verifySlotFile(*this, - "belongs in bucket", - "Document not belonging there", 9); - CPPUNIT_ASSERT_EQUAL(3u, _memFile->getBlobCount()); -*/ - } -} - -} -} diff --git a/memfilepersistence/src/tests/spi/options_builder.h b/memfilepersistence/src/tests/spi/options_builder.h deleted file mode 100644 index 7f04a02086c..00000000000 --- a/memfilepersistence/src/tests/spi/options_builder.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include -#include -#include - -namespace storage { -namespace memfile { - -class OptionsBuilder -{ - Options _newOptions; -public: - OptionsBuilder(const Options& opts) - : _newOptions(opts) - { - } - - OptionsBuilder& maximumReadThroughGap(uint32_t readThroughGap) { - _newOptions._maximumGapToReadThrough = readThroughGap; - return *this; - } - - OptionsBuilder& initialIndexRead(uint32_t bytesToRead) { - _newOptions._initialIndexRead = bytesToRead; - return *this; - } - - OptionsBuilder& revertTimePeriod(framework::MicroSecTime revertTime) { - _newOptions._revertTimePeriod = revertTime; - return *this; - } - - OptionsBuilder& defaultRemoveDocType(vespalib::stringref typeName) { - _newOptions._defaultRemoveDocType = typeName; - return *this; - } - - OptionsBuilder& maxDocumentVersions(uint32_t maxVersions) { - _newOptions._maxDocumentVersions = maxVersions; - return *this; - } - - std::unique_ptr build() const { - return std::unique_ptr(new Options(_newOptions)); - } -}; - -} // memfile -} // storage - diff --git a/memfilepersistence/src/tests/spi/providerconformancetest.cpp b/memfilepersistence/src/tests/spi/providerconformancetest.cpp deleted file mode 100644 index 7ba91bde619..00000000000 --- a/memfilepersistence/src/tests/spi/providerconformancetest.cpp +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "memfiletestutils.h" -#include - -namespace storage { -namespace memfile { - -struct ProviderConformanceTest : public spi::ConformanceTest { - struct Factory : public PersistenceFactory { - framework::defaultimplementation::ComponentRegisterImpl _compRegister; - framework::defaultimplementation::RealClock _clock; - std::unique_ptr cache; - - Factory() - : _compRegister(), - _clock() - { - _compRegister.setClock(_clock); - } - - spi::PersistenceProvider::UP - getPersistenceImplementation(const std::shared_ptr& repo, - const document::DocumenttypesConfig&) override - { - system("rm -rf vdsroot"); - system("mkdir -p vdsroot/disks/d0"); - vdstestlib::DirConfig config(getStandardConfig(true)); - - MemFilePersistenceProvider::UP result( - new MemFilePersistenceProvider( - _compRegister, - config.getConfigId())); - result->setDocumentRepo(*repo); - return spi::PersistenceProvider::UP(result.release()); - } - - bool - supportsRevert() const - { - return true; - } - }; - - ProviderConformanceTest() - : spi::ConformanceTest(PersistenceFactory::UP(new Factory)) {} - - CPPUNIT_TEST_SUITE(ProviderConformanceTest); - DEFINE_CONFORMANCE_TESTS(); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(ProviderConformanceTest); - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/spi/shared_data_location_tracker_test.cpp b/memfilepersistence/src/tests/spi/shared_data_location_tracker_test.cpp deleted file mode 100644 index ee95cc0026d..00000000000 --- a/memfilepersistence/src/tests/spi/shared_data_location_tracker_test.cpp +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include - -namespace storage { -namespace memfile { - -class SharedDataLocationTrackerTest : public CppUnit::TestFixture -{ -public: - void headerIsPassedDownToCacheAccessor(); - void bodyIsPassedDownToCacheAccessor(); - void firstInvocationReturnsNewLocation(); - void multipleInvocationsForSharedSlotReturnSameLocation(); - - CPPUNIT_TEST_SUITE(SharedDataLocationTrackerTest); - CPPUNIT_TEST(headerIsPassedDownToCacheAccessor); - CPPUNIT_TEST(bodyIsPassedDownToCacheAccessor); - CPPUNIT_TEST(firstInvocationReturnsNewLocation); - CPPUNIT_TEST(multipleInvocationsForSharedSlotReturnSameLocation); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(SharedDataLocationTrackerTest); - -namespace { - -using Params = std::pair; -constexpr auto HEADER = Types::HEADER; -constexpr auto BODY = Types::BODY; - -/** - * A simple mock of a buffer cache which records all invocations - * and returns a location increasing by 100 for each invocation. - */ -struct MockBufferCacheCopier : BufferCacheCopier -{ - // This is practically _screaming_ for GoogleMock. - std::vector invocations; - - DataLocation doCopyFromSourceToLocal( - Types::DocumentPart part, - DataLocation sourceLocation) override - { - Params params(part, sourceLocation); - const size_t invocationsBefore = invocations.size(); - invocations.push_back(params); - return DataLocation(invocationsBefore * 100, - invocationsBefore * 100 + 100); - } -}; - -} - -void -SharedDataLocationTrackerTest::headerIsPassedDownToCacheAccessor() -{ - MockBufferCacheCopier cache; - SharedDataLocationTracker tracker(cache, HEADER); - tracker.getOrCreateSharedLocation({0, 100}); - CPPUNIT_ASSERT_EQUAL(size_t(1), cache.invocations.size()); - CPPUNIT_ASSERT_EQUAL(Params(HEADER, {0, 100}), cache.invocations[0]); -} - -void -SharedDataLocationTrackerTest::bodyIsPassedDownToCacheAccessor() -{ - MockBufferCacheCopier cache; - SharedDataLocationTracker tracker(cache, BODY); - tracker.getOrCreateSharedLocation({0, 100}); - CPPUNIT_ASSERT_EQUAL(size_t(1), cache.invocations.size()); - CPPUNIT_ASSERT_EQUAL(Params(BODY, {0, 100}), cache.invocations[0]); -} - -void -SharedDataLocationTrackerTest::firstInvocationReturnsNewLocation() -{ - MockBufferCacheCopier cache; - SharedDataLocationTracker tracker(cache, HEADER); - // Auto-incrementing per cache copy invocation. - CPPUNIT_ASSERT_EQUAL(DataLocation(0, 100), - tracker.getOrCreateSharedLocation({500, 600})); - CPPUNIT_ASSERT_EQUAL(DataLocation(100, 200), - tracker.getOrCreateSharedLocation({700, 800})); - - CPPUNIT_ASSERT_EQUAL(size_t(2), cache.invocations.size()); - CPPUNIT_ASSERT_EQUAL(Params(HEADER, {500, 600}), cache.invocations[0]); - CPPUNIT_ASSERT_EQUAL(Params(HEADER, {700, 800}), cache.invocations[1]); -} - -void -SharedDataLocationTrackerTest - ::multipleInvocationsForSharedSlotReturnSameLocation() -{ - MockBufferCacheCopier cache; - SharedDataLocationTracker tracker(cache, HEADER); - CPPUNIT_ASSERT_EQUAL(DataLocation(0, 100), - tracker.getOrCreateSharedLocation({500, 600})); - // Same source location, thus we can reuse the same destination location - // as well. - CPPUNIT_ASSERT_EQUAL(DataLocation(0, 100), - tracker.getOrCreateSharedLocation({500, 600})); - - CPPUNIT_ASSERT_EQUAL(size_t(1), cache.invocations.size()); - CPPUNIT_ASSERT_EQUAL(Params(HEADER, {500, 600}), cache.invocations[0]); -} - -} // memfile -} // storage - diff --git a/memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp b/memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp deleted file mode 100644 index 7701df98661..00000000000 --- a/memfilepersistence/src/tests/spi/simplememfileiobuffertest.cpp +++ /dev/null @@ -1,656 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include - -namespace storage { -namespace memfile { - -class SimpleMemFileIOBufferTest : public SingleDiskMemFileTestUtils -{ - CPPUNIT_TEST_SUITE(SimpleMemFileIOBufferTest); - CPPUNIT_TEST(testAddAndReadDocument); - CPPUNIT_TEST(testNonExistingLocation); - CPPUNIT_TEST(testCopy); - CPPUNIT_TEST(testCacheLocation); - CPPUNIT_TEST(testPersist); - CPPUNIT_TEST(testGetSerializedSize); - CPPUNIT_TEST(testRemapLocations); - CPPUNIT_TEST(testAlignmentUtilFunctions); - CPPUNIT_TEST(testCalculatedCacheSize); - CPPUNIT_TEST(testSharedBuffer); - CPPUNIT_TEST(testSharedBufferUsage); - CPPUNIT_TEST(testHeaderChunkEncoderComputesSizesCorrectly); - CPPUNIT_TEST(testHeaderChunkEncoderSerializesIdCorrectly); - CPPUNIT_TEST(testHeaderChunkEncoderSerializesHeaderCorrectly); - CPPUNIT_TEST(testRemovesCanBeWrittenWithBlankDefaultDocument); - CPPUNIT_TEST(testRemovesCanBeWrittenWithIdInferredDoctype); - CPPUNIT_TEST(testRemovesWithInvalidDocTypeThrowsException); - CPPUNIT_TEST_SUITE_END(); - - using BufferType = SimpleMemFileIOBuffer::BufferType; - using BufferSP = BufferType::SP; - using BufferAllocation = SimpleMemFileIOBuffer::BufferAllocation; - using HeaderChunkEncoder = SimpleMemFileIOBuffer::HeaderChunkEncoder; - using SimpleMemFileIOBufferUP = std::unique_ptr; - - BufferAllocation allocateBuffer(size_t sz) { - return BufferAllocation(BufferSP(new BufferType(sz)), 0, sz); - } - - /** - * Create an I/O buffer instance with for a dummy bucket. If removeDocType - * is non-empty, remove entries will be written in backwards compatible - * mode. - */ - SimpleMemFileIOBufferUP createIoBufferWithDummySpec( - vespalib::stringref removeDocType = ""); - -public: - class DummyFileReader : public VersionSerializer { - public: - FileVersion getFileVersion() override { return FileVersion(); } - void loadFile(MemFile&, Environment&, Buffer&, uint64_t ) override {} - FlushResult flushUpdatesToFile(MemFile&, Environment&) override { - return FlushResult::TooSmall; - } - void rewriteFile(MemFile&, Environment&) override {} - bool verify(MemFile&, Environment&, std::ostream&, bool, uint16_t) override { return false; }; - void cacheLocations(MemFileIOInterface&, Environment&, const Options&, - DocumentPart, const std::vector&) override {} - }; - - DummyFileReader dfr; - - void testAddAndReadDocument(); - void testNonExistingLocation(); - void testCopy(); - void testCacheLocation(); - void testPersist(); - void testGetSerializedSize(); - void testRemapLocations(); - void testAlignmentUtilFunctions(); - void testCalculatedCacheSize(); - void testSharedBuffer(); - void testSharedBufferUsage(); - void testHeaderChunkEncoderComputesSizesCorrectly(); - void testHeaderChunkEncoderSerializesIdCorrectly(); - void testHeaderChunkEncoderSerializesHeaderCorrectly(); - void testRemovesCanBeWrittenWithBlankDefaultDocument(); - void testRemovesCanBeWrittenWithIdInferredDoctype(); - void testRemovesWithInvalidDocTypeThrowsException(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(SimpleMemFileIOBufferTest); - - -void -SimpleMemFileIOBufferTest::testAddAndReadDocument() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 456, - 789, - 1234)); - - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr(new FileInfo), - fileSpec, - env()); - - DataLocation h = buffer.addHeader(*doc); - DataLocation b = buffer.addBody(*doc); - - Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), h); - buffer.readBody(*getTypeRepo(), b, *newDoc); - - CPPUNIT_ASSERT_EQUAL(*doc, *newDoc); - CPPUNIT_ASSERT_EQUAL(true, buffer.isCached(h, HEADER)); - CPPUNIT_ASSERT_EQUAL(true, buffer.isCached(b, BODY)); - CPPUNIT_ASSERT_EQUAL(false, buffer.isCached(h, BODY)); - CPPUNIT_ASSERT_EQUAL(false, buffer.isCached(b, HEADER)); - CPPUNIT_ASSERT_EQUAL(doc->getId(), buffer.getDocumentId(h)); -} - -void -SimpleMemFileIOBufferTest::testPersist() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 456, - 789, - 1234)); - - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr(new FileInfo), - fileSpec, - env()); - - DataLocation h = buffer.addHeader(*doc); - DataLocation b = buffer.addBody(*doc); - - CPPUNIT_ASSERT(!buffer.isPersisted(h, HEADER)); - CPPUNIT_ASSERT(!buffer.isPersisted(b, BODY)); - - buffer.persist(HEADER, h, DataLocation(1000, h.size())); - buffer.persist(BODY, b, DataLocation(5000, b.size())); - - Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), DataLocation(1000, h.size())); - buffer.readBody(*getTypeRepo(), DataLocation(5000, b.size()), *newDoc); - - CPPUNIT_ASSERT(buffer.isPersisted(DataLocation(1000, h.size()), HEADER)); - CPPUNIT_ASSERT(buffer.isPersisted(DataLocation(5000, b.size()), BODY)); - - CPPUNIT_ASSERT_EQUAL(*doc, *newDoc); -} - -void -SimpleMemFileIOBufferTest::testCopy() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr(new FileInfo), - fileSpec, - env()); - - for (uint32_t i = 0; i < 10; ++i) { - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 456, - 789, - 1234)); - - DataLocation h = buffer.addHeader(*doc); - DataLocation b = buffer.addBody(*doc); - - SimpleMemFileIOBuffer buffer2(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr(new FileInfo), - fileSpec, - env()); - - DataLocation h2 = buffer2.copyCache(buffer, HEADER, h); - DataLocation b2 = buffer2.copyCache(buffer, BODY, b); - - Document::UP newDoc = buffer2.getDocumentHeader(*getTypeRepo(), h2); - buffer2.readBody(*getTypeRepo(), b2, *newDoc); - - CPPUNIT_ASSERT_EQUAL(*doc, *newDoc); - } -} - -void -SimpleMemFileIOBufferTest::testNonExistingLocation() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 456, - 789, - 1234)); - - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr(new FileInfo), - fileSpec, - env()); - - DataLocation h = buffer.addHeader(*doc); - DataLocation b = buffer.addBody(*doc); - - buffer.clear(HEADER); - - try { - Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), h); - CPPUNIT_ASSERT(false); - } catch (SimpleMemFileIOBuffer::PartNotCachedException& e) { - } - - buffer.clear(BODY); - - try { - document::Document newDoc; - buffer.readBody(*getTypeRepo(), b, newDoc); - CPPUNIT_ASSERT(false); - } catch (SimpleMemFileIOBuffer::PartNotCachedException& e) { - } -} - -void -SimpleMemFileIOBufferTest::testCacheLocation() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - FileInfo::UP(new FileInfo(100, 10000, 50000)), - fileSpec, - env()); - - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 456, - 789, - 1234)); - - BufferAllocation headerBuf = buffer.serializeHeader(*doc); - BufferAllocation bodyBuf = buffer.serializeBody(*doc); - - DataLocation hloc(1234, headerBuf.getSize()); - DataLocation bloc(5678, bodyBuf.getSize()); - - buffer.cacheLocation(HEADER, hloc, headerBuf.getSharedBuffer(), 0); - buffer.cacheLocation(BODY, bloc, bodyBuf.getSharedBuffer(), 0); - - Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), hloc); - buffer.readBody(*getTypeRepo(), bloc, *newDoc); - - CPPUNIT_ASSERT_EQUAL(*doc, *newDoc); -} - -void -SimpleMemFileIOBufferTest::testGetSerializedSize() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - FileInfo::UP(new FileInfo(100, 10000, 50000)), - fileSpec, - env()); - - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 456, - 789, - 1234)); - - BufferAllocation headerBuf = buffer.serializeHeader(*doc); - BufferAllocation bodyBuf = buffer.serializeBody(*doc); - - DataLocation hloc(1234, headerBuf.getSize()); - DataLocation bloc(5678, bodyBuf.getSize()); - - buffer.cacheLocation(HEADER, hloc, headerBuf.getSharedBuffer(), 0); - buffer.cacheLocation(BODY, bloc, bodyBuf.getSharedBuffer(), 0); - - vespalib::nbostream serializedHeader; - doc->serializeHeader(serializedHeader); - - vespalib::nbostream serializedBody; - doc->serializeBody(serializedBody); - - CPPUNIT_ASSERT_EQUAL(uint32_t(serializedHeader.size()), - buffer.getSerializedSize(HEADER, hloc)); - CPPUNIT_ASSERT_EQUAL(uint32_t(serializedBody.size()), - buffer.getSerializedSize(BODY, bloc)); -} - -// Test that remapping does not overwrite datalocations that it has -// already updated -void -SimpleMemFileIOBufferTest::testRemapLocations() -{ - FileSpecification fileSpec(BucketId(16, 123), env().getDirectory(), "testfile.0"); - - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - FileInfo::UP(new FileInfo(100, 10000, 50000)), - fileSpec, - env()); - - document::Document::SP doc(createRandomDocumentAtLocation( - 123, - 100, - 100)); - BufferAllocation headerBuf = buffer.serializeHeader(*doc); - BufferAllocation bodyBuf = buffer.serializeBody(*doc); - - document::Document::SP doc2(createRandomDocumentAtLocation( - 123, - 100, - 100)); - - BufferAllocation headerBuf2 = buffer.serializeHeader(*doc2); - BufferAllocation bodyBuf2 = buffer.serializeBody(*doc2); - - DataLocation hloc(30000, headerBuf.getSize()); - DataLocation hloc2(0, headerBuf2.getSize()); - DataLocation hloc3(10000, hloc2._size); - - buffer.cacheLocation(HEADER, hloc, headerBuf.getSharedBuffer(), 0); - buffer.cacheLocation(HEADER, hloc2, headerBuf2.getSharedBuffer(), 0); - - std::map remapping; - remapping[hloc2] = hloc; - remapping[hloc] = hloc3; - - buffer.remapAndPersistAllLocations(HEADER, remapping); - - Document::UP newDoc = buffer.getDocumentHeader(*getTypeRepo(), hloc3); - document::ByteBuffer bbuf(bodyBuf.getBuffer(), bodyBuf.getSize()); - newDoc->deserializeBody(*getTypeRepo(), bbuf); - - CPPUNIT_ASSERT_EQUAL(*doc, *newDoc); - - Document::UP newDoc2 = buffer.getDocumentHeader(*getTypeRepo(), hloc); - document::ByteBuffer bbuf2(bodyBuf.getBuffer(), bodyBuf.getSize()); - newDoc2->deserializeBody(*getTypeRepo(), bbuf2); - CPPUNIT_ASSERT_EQUAL(*doc2, *newDoc2); -} - -/** - * Not technically a part of SimpleMemFileIOBuffer, but used by it and - * currently contained within its header file. Move test somewhere else - * if the code itself is moved. - */ -void -SimpleMemFileIOBufferTest::testAlignmentUtilFunctions() -{ - using namespace util; - CPPUNIT_ASSERT_EQUAL(size_t(0), alignUpPow2<4096>(0)); - CPPUNIT_ASSERT_EQUAL(size_t(4096), alignUpPow2<4096>(1)); - CPPUNIT_ASSERT_EQUAL(size_t(4096), alignUpPow2<4096>(512)); - CPPUNIT_ASSERT_EQUAL(size_t(4096), alignUpPow2<4096>(4096)); - CPPUNIT_ASSERT_EQUAL(size_t(8192), alignUpPow2<4096>(4097)); - CPPUNIT_ASSERT_EQUAL(size_t(32), alignUpPow2<16>(20)); - CPPUNIT_ASSERT_EQUAL(size_t(32), alignUpPow2<32>(20)); - CPPUNIT_ASSERT_EQUAL(size_t(64), alignUpPow2<64>(20)); - CPPUNIT_ASSERT_EQUAL(size_t(128), alignUpPow2<128>(20)); - - CPPUNIT_ASSERT_EQUAL(uint32_t(0), nextPow2(0)); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), nextPow2(1)); - CPPUNIT_ASSERT_EQUAL(uint32_t(4), nextPow2(3)); - CPPUNIT_ASSERT_EQUAL(uint32_t(16), nextPow2(15)); - CPPUNIT_ASSERT_EQUAL(uint32_t(64), nextPow2(40)); - CPPUNIT_ASSERT_EQUAL(uint32_t(64), nextPow2(64)); -} - -/** - * Test that allocated buffers are correctly reported with their sizes - * rounded up to account for mmap overhead. - */ -void -SimpleMemFileIOBufferTest::testCalculatedCacheSize() -{ - FileSpecification fileSpec(BucketId(16, 123), - env().getDirectory(), "testfile.0"); - SimpleMemFileIOBuffer buffer(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr(new FileInfo), - fileSpec, - env()); - - CPPUNIT_ASSERT_EQUAL(size_t(0), buffer.getCachedSize(HEADER)); - CPPUNIT_ASSERT_EQUAL(size_t(0), buffer.getCachedSize(BODY)); - - // All buffers are on a 4k page granularity. - BufferAllocation sharedHeaderBuffer(allocateBuffer(1500)); // -> 4096 - buffer.cacheLocation(HEADER, DataLocation(0, 85), - sharedHeaderBuffer.getSharedBuffer(), 0); - CPPUNIT_ASSERT_EQUAL(size_t(4096), buffer.getCachedSize(HEADER)); - - buffer.cacheLocation(HEADER, DataLocation(200, 100), - sharedHeaderBuffer.getSharedBuffer(), 85); - CPPUNIT_ASSERT_EQUAL(size_t(4096), buffer.getCachedSize(HEADER)); - - BufferAllocation singleHeaderBuffer(allocateBuffer(200)); // -> 4096 - buffer.cacheLocation(HEADER, DataLocation(0, 100), - singleHeaderBuffer.getSharedBuffer(), 0); - CPPUNIT_ASSERT_EQUAL(size_t(8192), buffer.getCachedSize(HEADER)); - - BufferAllocation singleBodyBuffer(allocateBuffer(300)); // -> 4096 - buffer.cacheLocation(BODY, DataLocation(0, 100), - singleBodyBuffer.getSharedBuffer(), 0); - CPPUNIT_ASSERT_EQUAL(size_t(4096), buffer.getCachedSize(BODY)); - - buffer.clear(HEADER); - CPPUNIT_ASSERT_EQUAL(size_t(0), buffer.getCachedSize(HEADER)); - - buffer.clear(BODY); - CPPUNIT_ASSERT_EQUAL(size_t(0), buffer.getCachedSize(BODY)); -} - -void -SimpleMemFileIOBufferTest::testSharedBuffer() -{ - typedef SimpleMemFileIOBuffer::SharedBuffer SharedBuffer; - - { - SharedBuffer buf(1024); - CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getSize()); - CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.getUsedSize()); - CPPUNIT_ASSERT(buf.hasRoomFor(1024)); - CPPUNIT_ASSERT(!buf.hasRoomFor(1025)); - - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.allocate(13)); - // Allocation should be rounded up to nearest alignment. - // TODO: is this even necessary? - CPPUNIT_ASSERT_EQUAL(size_t(16), buf.getUsedSize()); - CPPUNIT_ASSERT_EQUAL(size_t(1008), buf.getFreeSize()); - CPPUNIT_ASSERT(buf.hasRoomFor(1008)); - CPPUNIT_ASSERT(!buf.hasRoomFor(1009)); - CPPUNIT_ASSERT_EQUAL(size_t(16), buf.allocate(1)); - CPPUNIT_ASSERT_EQUAL(size_t(24), buf.getUsedSize()); - - CPPUNIT_ASSERT_EQUAL(size_t(24), buf.allocate(999)); - CPPUNIT_ASSERT(!buf.hasRoomFor(1)); - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getUsedSize()); - } - // Test exact fit. - { - SharedBuffer buf(1024); - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.allocate(1024)); - CPPUNIT_ASSERT(!buf.hasRoomFor(1)); - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getUsedSize()); - } - // Test 512-byte alignment. - { - SharedBuffer buf(1024); - CPPUNIT_ASSERT(buf.hasRoomFor(1000, SharedBuffer::ALIGN_512_BYTES)); - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.allocate(10)); - CPPUNIT_ASSERT(!buf.hasRoomFor(1000, SharedBuffer::ALIGN_512_BYTES)); - CPPUNIT_ASSERT(!buf.hasRoomFor(513, SharedBuffer::ALIGN_512_BYTES)); - CPPUNIT_ASSERT(buf.hasRoomFor(512, SharedBuffer::ALIGN_512_BYTES)); - CPPUNIT_ASSERT_EQUAL(size_t(512), buf.allocate(512, SharedBuffer::ALIGN_512_BYTES)); - CPPUNIT_ASSERT_EQUAL(size_t(0), buf.getFreeSize()); - CPPUNIT_ASSERT_EQUAL(size_t(1024), buf.getUsedSize()); - } -} - -void -SimpleMemFileIOBufferTest::testSharedBufferUsage() -{ - FileSpecification fileSpec(BucketId(16, 123), - env().getDirectory(), "testfile.0"); - SimpleMemFileIOBuffer ioBuf(dfr, - vespalib::LazyFile::UP(), - std::unique_ptr(new FileInfo), - fileSpec, - env()); - - const size_t threshold = SimpleMemFileIOBuffer::WORKING_BUFFER_SIZE; - - // Brand new allocation - BufferAllocation ba(ioBuf.allocateBuffer(HEADER, 1)); - CPPUNIT_ASSERT(ba.buf.get()); - CPPUNIT_ASSERT_EQUAL(uint32_t(0), ba.pos); - CPPUNIT_ASSERT_EQUAL(uint32_t(1), ba.size); - // Should reuse buffer, but get other offset - BufferAllocation ba2(ioBuf.allocateBuffer(HEADER, 500)); - CPPUNIT_ASSERT_EQUAL(ba.buf.get(), ba2.buf.get()); - CPPUNIT_ASSERT_EQUAL(uint32_t(8), ba2.pos); - CPPUNIT_ASSERT_EQUAL(uint32_t(500), ba2.size); - CPPUNIT_ASSERT_EQUAL(size_t(512), ba2.buf->getUsedSize()); - - // Allocate a buffer so big that it should get its own buffer instance - BufferAllocation ba3(ioBuf.allocateBuffer(HEADER, threshold)); - CPPUNIT_ASSERT(ba3.buf.get() != ba2.buf.get()); - CPPUNIT_ASSERT_EQUAL(uint32_t(0), ba3.pos); - CPPUNIT_ASSERT_EQUAL(uint32_t(threshold), ba3.size); - - // But smaller allocs should still be done from working buffer - BufferAllocation ba4(ioBuf.allocateBuffer(HEADER, 512)); - CPPUNIT_ASSERT_EQUAL(ba.buf.get(), ba4.buf.get()); - CPPUNIT_ASSERT_EQUAL(uint32_t(512), ba4.pos); - CPPUNIT_ASSERT_EQUAL(uint32_t(512), ba4.size); - CPPUNIT_ASSERT_EQUAL(size_t(1024), ba4.buf->getUsedSize()); - - // Allocate lots of smaller buffers from the same buffer until we run out. - while (true) { - BufferAllocation tmp(ioBuf.allocateBuffer(HEADER, 1024)); - CPPUNIT_ASSERT_EQUAL(ba.buf.get(), tmp.buf.get()); - if (!tmp.buf->hasRoomFor(2048)) { - break; - } - } - BufferAllocation ba5(ioBuf.allocateBuffer(HEADER, 2048)); - CPPUNIT_ASSERT(ba5.buf.get() != ba.buf.get()); - CPPUNIT_ASSERT_EQUAL(uint32_t(0), ba5.pos); - CPPUNIT_ASSERT_EQUAL(uint32_t(2048), ba5.size); - - // Allocating for different part should get different buffer. - BufferAllocation ba6(ioBuf.allocateBuffer(BODY, 128)); - CPPUNIT_ASSERT(ba6.buf.get() != ba5.buf.get()); - CPPUNIT_ASSERT_EQUAL(uint32_t(0), ba6.pos); - CPPUNIT_ASSERT_EQUAL(uint32_t(128), ba6.size); -} - -void -SimpleMemFileIOBufferTest::testHeaderChunkEncoderComputesSizesCorrectly() -{ - document::Document::SP doc(createRandomDocumentAtLocation(123, 100, 100)); - - std::string idString = doc->getId().toString(); - HeaderChunkEncoder encoder(doc->getId()); - // Without document, payload is: 3x u32 + doc id string (no zero term). - CPPUNIT_ASSERT_EQUAL(sizeof(uint32_t)*3 + idString.size(), - static_cast(encoder.encodedSize())); - - encoder.bufferDocument(*doc); - vespalib::nbostream stream; - doc->serializeHeader(stream); - // With document, add size of serialized document to the mix. - CPPUNIT_ASSERT_EQUAL(sizeof(uint32_t)*3 + idString.size() + stream.size(), - static_cast(encoder.encodedSize())); -} - -SimpleMemFileIOBufferTest::SimpleMemFileIOBufferUP -SimpleMemFileIOBufferTest::createIoBufferWithDummySpec( - vespalib::stringref removeDocType) -{ - FileSpecification fileSpec(BucketId(16, 123), - env().getDirectory(), "testfile.0"); - // Override config. - auto options = env().acquireConfigReadLock().options(); - env().acquireConfigWriteLock().setOptions( - OptionsBuilder(*options) - .defaultRemoveDocType(removeDocType) - .build()); - - SimpleMemFileIOBufferUP ioBuf( - new SimpleMemFileIOBuffer( - dfr, - vespalib::LazyFile::UP(), - std::unique_ptr(new FileInfo), - fileSpec, - env())); - return ioBuf; -} - -void -SimpleMemFileIOBufferTest::testHeaderChunkEncoderSerializesIdCorrectly() -{ - document::Document::SP doc(createRandomDocumentAtLocation(123, 100, 100)); - HeaderChunkEncoder encoder(doc->getId()); - - SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec()); - - BufferAllocation buf(ioBuf->allocateBuffer(HEADER, encoder.encodedSize())); - encoder.writeTo(buf); - DataLocation newLoc = ioBuf->addLocation(HEADER, buf); - document::DocumentId checkId = ioBuf->getDocumentId(newLoc); - - CPPUNIT_ASSERT_EQUAL(doc->getId(), checkId); -} - -void -SimpleMemFileIOBufferTest::testHeaderChunkEncoderSerializesHeaderCorrectly() -{ - document::Document::SP doc(createRandomDocumentAtLocation(123, 100, 100)); - HeaderChunkEncoder encoder(doc->getId()); - encoder.bufferDocument(*doc); - - SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec()); - BufferAllocation buf(ioBuf->allocateBuffer(HEADER, encoder.encodedSize())); - encoder.writeTo(buf); - DataLocation newLoc = ioBuf->addLocation(HEADER, buf); - Document::UP checkDoc = ioBuf->getDocumentHeader(*getTypeRepo(), newLoc); - - CPPUNIT_ASSERT_EQUAL(doc->getId(), checkDoc->getId()); - CPPUNIT_ASSERT_EQUAL(doc->getType(), checkDoc->getType()); -} - -void -SimpleMemFileIOBufferTest::testRemovesCanBeWrittenWithBlankDefaultDocument() -{ - SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec("testdoctype1")); - - document::DocumentId id("userdoc:yarn:12345:fluff"); - DataLocation loc(ioBuf->addDocumentIdOnlyHeader(id, *getTypeRepo())); - // Despite adding with document id only, we should now actually have a - // valid document header. Will fail with a DeserializeException if no - // header has been written. - Document::UP removeWithHeader( - ioBuf->getDocumentHeader(*getTypeRepo(), loc)); - CPPUNIT_ASSERT_EQUAL(removeWithHeader->getId(), id); - CPPUNIT_ASSERT_EQUAL(removeWithHeader->getType(), - *getTypeRepo()->getDocumentType("testdoctype1")); -} - -void -SimpleMemFileIOBufferTest::testRemovesCanBeWrittenWithIdInferredDoctype() -{ - SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec("testdoctype1")); - - document::DocumentId id("id:yarn:testdoctype2:n=12345:fluff"); - DataLocation loc(ioBuf->addDocumentIdOnlyHeader(id, *getTypeRepo())); - // Since document id contains an explicit document type, the blank remove - // document header should be written with that type instead of the one - // provided as default via config. - Document::UP removeWithHeader( - ioBuf->getDocumentHeader(*getTypeRepo(), loc)); - CPPUNIT_ASSERT_EQUAL(removeWithHeader->getId(), id); - CPPUNIT_ASSERT_EQUAL(removeWithHeader->getType(), - *getTypeRepo()->getDocumentType("testdoctype2")); -} - -void -SimpleMemFileIOBufferTest::testRemovesWithInvalidDocTypeThrowsException() -{ - SimpleMemFileIOBufferUP ioBuf(createIoBufferWithDummySpec("testdoctype1")); - - document::DocumentId id("id:yarn:nosuchtype:n=12345:fluff"); - try { - DataLocation loc(ioBuf->addDocumentIdOnlyHeader(id, *getTypeRepo())); - CPPUNIT_FAIL("No exception thrown on bad doctype"); - } catch (const vespalib::Exception& e) { - CPPUNIT_ASSERT(e.getMessage().find("Could not serialize document " - "for remove with unknown doctype " - "'nosuchtype'") - != std::string::npos); - } -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/spi/simulatedfailurefile.cpp b/memfilepersistence/src/tests/spi/simulatedfailurefile.cpp deleted file mode 100644 index b7da294f8eb..00000000000 --- a/memfilepersistence/src/tests/spi/simulatedfailurefile.cpp +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "simulatedfailurefile.h" -#include - -namespace storage { -namespace memfile { - -vespalib::LazyFile::UP -SimulatedFailureLazyFile::Factory::createFile(const std::string& fileName) const { - return vespalib::LazyFile::UP( - new SimulatedFailureLazyFile(fileName, - vespalib::File::DIRECTIO, - _readOpsBeforeFailure, - _writeOpsBeforeFailure)); -} - -SimulatedFailureLazyFile::SimulatedFailureLazyFile( - const std::string& filename, - int flags, - int readOpsBeforeFailure, - int writeOpsBeforeFailure) - : LazyFile(filename, flags), - _readOpsBeforeFailure(readOpsBeforeFailure), - _writeOpsBeforeFailure(writeOpsBeforeFailure) -{ -} - -off_t -SimulatedFailureLazyFile::write(const void *buf, size_t bufsize, off_t offset) -{ - if (_writeOpsBeforeFailure == 0) { - throw vespalib::IoException( - "A simulated I/O write exception was triggered", - vespalib::IoException::CORRUPT_DATA, VESPA_STRLOC); - } - --_writeOpsBeforeFailure; - return vespalib::LazyFile::write(buf, bufsize, offset); -} - -size_t -SimulatedFailureLazyFile::read(void *buf, size_t bufsize, off_t offset) const -{ - if (_readOpsBeforeFailure == 0) { - throw vespalib::IoException( - "A simulated I/O read exception was triggered", - vespalib::IoException::CORRUPT_DATA, VESPA_STRLOC); - } - --_readOpsBeforeFailure; - return vespalib::LazyFile::read(buf, bufsize, offset); -} - -} // ns memfile -} // ns storage - diff --git a/memfilepersistence/src/tests/spi/simulatedfailurefile.h b/memfilepersistence/src/tests/spi/simulatedfailurefile.h deleted file mode 100644 index e3dbd5e13e2..00000000000 --- a/memfilepersistence/src/tests/spi/simulatedfailurefile.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include -#include - -namespace storage { -namespace memfile { - -class SimulatedFailureLazyFile : public vespalib::LazyFile -{ - mutable int _readOpsBeforeFailure; - mutable int _writeOpsBeforeFailure; -public: - class Factory : public Environment::LazyFileFactory { - public: - Factory() - : _readOpsBeforeFailure(-1), - _writeOpsBeforeFailure(0) - { } - vespalib::LazyFile::UP createFile(const std::string& fileName) const override; - - void setReadOpsBeforeFailure(int ops) { - _readOpsBeforeFailure = ops; - } - - void setWriteOpsBeforeFailure(int ops) { - _writeOpsBeforeFailure = ops; - } - private: - int _readOpsBeforeFailure; - int _writeOpsBeforeFailure; - }; - - SimulatedFailureLazyFile( - const std::string& filename, - int flags, - int readOpsBeforeFailure, - int writeOpsBeforeFailure); - - off_t write(const void *buf, size_t bufsize, off_t offset) override; - size_t read(void *buf, size_t bufsize, off_t offset) const override; -}; - -} // ns memfile -} // ns storage - diff --git a/memfilepersistence/src/tests/spi/splitoperationhandlertest.cpp b/memfilepersistence/src/tests/spi/splitoperationhandlertest.cpp deleted file mode 100644 index 89a7d0f6e03..00000000000 --- a/memfilepersistence/src/tests/spi/splitoperationhandlertest.cpp +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "memfiletestutils.h" -#include -#include - -using document::DocumentType; -using storage::spi::test::makeSpiBucket; - -namespace storage { -namespace memfile { -namespace { - spi::LoadType defaultLoadType(0, "default"); -} - -class SplitOperationHandlerTest : public SingleDiskMemFileTestUtils -{ - - void doTestMultiDisk(uint16_t sourceDisk, - uint16_t targetDisk0, - uint16_t targetDisk1); - - - CPPUNIT_TEST_SUITE(SplitOperationHandlerTest); - CPPUNIT_TEST(testSimple); - CPPUNIT_TEST(testMultiDisk); - CPPUNIT_TEST(testMultiDiskNonZeroSourceIndex); - CPPUNIT_TEST(testExceptionDuringSplittingEvictsAllBuckets); - CPPUNIT_TEST_SUITE_END(); - -public: - void testSimple(); - void testMultiDisk(); - void testMultiDiskNonZeroSourceIndex(); - void testExceptionDuringSplittingEvictsAllBuckets(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(SplitOperationHandlerTest); - -void -SplitOperationHandlerTest::testSimple() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(1); - - for (uint32_t i = 0; i < 100; i++) { - uint32_t location = 4; - if (i % 2 == 0) { - location |= (1 << 16); - } - - doPut(location, Timestamp(1000 + i)); - } - flush(document::BucketId(16, 4)); - - env()._cache.clear(); - - document::BucketId sourceBucket = document::BucketId(16, 4); - document::BucketId target1 = document::BucketId(17, 4); - document::BucketId target2 = document::BucketId(17, 4 | (1 << 16)); - - SplitOperationHandler handler(env()); - spi::Result result = getPersistenceProvider().split( - makeSpiBucket(sourceBucket), - makeSpiBucket(target1), - makeSpiBucket(target2), - context); - - env()._cache.clear(); - - { - MemFilePtr file(handler.getMemFile(sourceBucket, 0)); - CPPUNIT_ASSERT_EQUAL(0, (int)file->getSlotCount()); - } - - { - MemFilePtr file(handler.getMemFile(target1, 0)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - for (uint32_t i = 0; i < file->getSlotCount(); ++i) { - file->getDocument((*file)[i], ALL); - } - } - - { - MemFilePtr file(handler.getMemFile(target2, 0)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - for (uint32_t i = 0; i < file->getSlotCount(); ++i) { - file->getDocument((*file)[i], ALL); - } - } -} - -void -SplitOperationHandlerTest::doTestMultiDisk(uint16_t sourceDisk, - uint16_t targetDisk0, - uint16_t targetDisk1) -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(3); - - for (uint32_t i = 0; i < 100; i++) { - uint32_t location = 4; - if (i % 2 == 0) { - location |= (1 << 16); - } - - doPutOnDisk(sourceDisk, location, Timestamp(1000 + i)); - } - flush(document::BucketId(16, 4)); - - env()._cache.clear(); - - document::BucketId sourceBucket = document::BucketId(16, 4); - document::BucketId target1 = document::BucketId(17, 4); - document::BucketId target2 = document::BucketId(17, 4 | (1 << 16)); - - SplitOperationHandler handler(env()); - spi::Result result = getPersistenceProvider().split( - makeSpiBucket(sourceBucket, spi::PartitionId(sourceDisk)), - makeSpiBucket(target1, spi::PartitionId(targetDisk0)), - makeSpiBucket(target2, spi::PartitionId(targetDisk1)), - context); - - env()._cache.clear(); - - { - MemFilePtr file(handler.getMemFile(sourceBucket, sourceDisk)); - CPPUNIT_ASSERT_EQUAL(0, (int)file->getSlotCount()); - } - - { - MemFilePtr file(handler.getMemFile(target1, targetDisk0)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - for (uint32_t i = 0; i < file->getSlotCount(); ++i) { - file->getDocument((*file)[i], ALL); - } - } - - { - MemFilePtr file(handler.getMemFile(target2, targetDisk1)); - CPPUNIT_ASSERT_EQUAL(50, (int)file->getSlotCount()); - for (uint32_t i = 0; i < file->getSlotCount(); ++i) { - file->getDocument((*file)[i], ALL); - } - } -} - -void -SplitOperationHandlerTest::testMultiDisk() -{ - doTestMultiDisk(0, 1, 2); -} - -void -SplitOperationHandlerTest::testMultiDiskNonZeroSourceIndex() -{ - doTestMultiDisk(1, 2, 0); -} - -void -SplitOperationHandlerTest::testExceptionDuringSplittingEvictsAllBuckets() -{ - spi::Context context(defaultLoadType, spi::Priority(0), - spi::Trace::TraceLevel(0)); - setupDisks(1); - - for (uint32_t i = 0; i < 100; i++) { - uint32_t location = 4; - if (i % 2 == 0) { - location |= (1 << 16); - } - - doPut(location, Timestamp(1000 + i)); - } - flush(document::BucketId(16, 4)); - - simulateIoErrorsForSubsequentlyOpenedFiles(); - - document::BucketId sourceBucket(16, 4); - document::BucketId target1(17, 4); - document::BucketId target2(17, 4 | (1 << 16)); - - try { - SplitOperationHandler handler(env()); - spi::Result result = getPersistenceProvider().split( - makeSpiBucket(sourceBucket), - makeSpiBucket(target1), - makeSpiBucket(target2), - context); - CPPUNIT_FAIL("Exception not thrown on flush failure"); - } catch (std::exception&) { - } - - CPPUNIT_ASSERT(!env()._cache.contains(sourceBucket)); - CPPUNIT_ASSERT(!env()._cache.contains(target1)); - CPPUNIT_ASSERT(!env()._cache.contains(target2)); - - unSimulateIoErrorsForSubsequentlyOpenedFiles(); - - // Source must not have been deleted - { - SplitOperationHandler handler(env()); - MemFilePtr file(handler.getMemFile(sourceBucket, 0)); - CPPUNIT_ASSERT_EQUAL(100, (int)file->getSlotCount()); - } -} - -} - -} diff --git a/memfilepersistence/src/tests/testrunner.cpp b/memfilepersistence/src/tests/testrunner.cpp deleted file mode 100644 index 7bd12fedce5..00000000000 --- a/memfilepersistence/src/tests/testrunner.cpp +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include - -#include -LOG_SETUP("persistencecppunittests"); - -int -main(int argc, const char *argv[]) -{ - vdstestlib::CppUnitTestRunner testRunner; - return testRunner.run(argc, argv); -} diff --git a/memfilepersistence/src/tests/tools/.gitignore b/memfilepersistence/src/tests/tools/.gitignore deleted file mode 100644 index 7e7c0fe7fae..00000000000 --- a/memfilepersistence/src/tests/tools/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.depend -/Makefile diff --git a/memfilepersistence/src/tests/tools/CMakeLists.txt b/memfilepersistence/src/tests/tools/CMakeLists.txt deleted file mode 100644 index fdbe5f4cc01..00000000000 --- a/memfilepersistence/src/tests/tools/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(memfilepersistence_testtools - SOURCES - dumpslotfiletest.cpp - vdsdisktooltest.cpp - DEPENDS - memfilepersistence_testspi - memfilepersistence -) diff --git a/memfilepersistence/src/tests/tools/dumpslotfiletest.cpp b/memfilepersistence/src/tests/tools/dumpslotfiletest.cpp deleted file mode 100644 index 92c0736a10a..00000000000 --- a/memfilepersistence/src/tests/tools/dumpslotfiletest.cpp +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace storage { -namespace memfile { - -class DumpSlotFileTest : public SingleDiskMemFileTestUtils -{ - CPPUNIT_TEST_SUITE(DumpSlotFileTest); - CPPUNIT_TEST(testSimple); - CPPUNIT_TEST_SUITE_END(); - -public: - void testSimple(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(DumpSlotFileTest); - -#define ASSERT_MATCH(optstring, pattern) \ -{ \ - vespalib::AppOptions opts("dumpslotfile " optstring); \ - std::ostringstream out; \ - config::ConfigUri configUri(config::ConfigUri::createFromInstance( \ - document::TestDocRepo::getDefaultConfig())); \ - std::unique_ptr config = config::ConfigGetter::getConfig(configUri.getConfigId(), configUri.getContext()); \ - SlotFileDumper::dump(opts.getArgCount(), opts.getArguments(), \ - configUri, out, out); \ - CPPUNIT_ASSERT_MATCH_REGEX(pattern, out.str()); \ - output = out.str(); \ -} - -void -DumpSlotFileTest::testSimple() -{ - std::string output; - // Test syntax page - ASSERT_MATCH("--help", ".*Usage: dumpslotfile.*"); - // Test non-existing file. (Handle as empty file) - ASSERT_MATCH("00a.0", - ".*BucketId\\(0x000000000000000a\\)" - ".*document count: 0.*non-existing.*"); - // Parse bucketid without extension. - ASSERT_MATCH("000000000000000a", - ".*BucketId\\(0x000000000000000a\\) " - "\\(extracted from filename\\).*"); - // Parse invalid bucket id. - ASSERT_MATCH("000010000000000g", - ".*Failed to extract bucket id from filename.*"); - // Test toXml with no data. Thus doesn't require doc config - ASSERT_MATCH("--toxml --documentconfig whatevah 000a.0", - ".*.*"); - // Test invalid arguments - ASSERT_MATCH("--foobar", ".*Invalid option 'foobar'\\..*"); - // What to show in XML doesn't make sense in non-xml mode - ASSERT_MATCH("--includeremoveddocs 0.0", - ".*Options for what to include in XML makes no sense when not " - "printing XML content.*"); - ASSERT_MATCH("--includeremoveentries 0.0", - ".*Options for what to include in XML makes no sense when not " - "printing XML content.*"); - // To binary only works for single doc - ASSERT_MATCH("--tobinary 0.0", - ".*To binary option only works for a single document.*"); - - BucketId bid(1, 0); - createTestBucket(bid, 0); - ASSERT_MATCH("-nN vdsroot/disks/d0/400000000000000.0", - ".*" - "Unique document count: 8.*" - "Total document size: [0-9]+.*" - "Used size: [0-9]+.*" - "Filename: .*/d0/.*" - "Filesize: 12288.*" - "SlotFileHeader.*" - "[0-9]+ empty entries.*" - "Header block.*" - "Content block.*" - "Slotfile verified.*" - ); - ASSERT_MATCH("vdsroot/disks/d0/400000000000000.0", ".*ff ff ff ff.*"); - - // User friendly output - ASSERT_MATCH("--friendly -nN vdsroot/disks/d0/400000000000000.0", - ".*id:mail:testdoctype1:n=0:9380.html.*"); - - ASSERT_MATCH("--tobinary " - "--docid id:mail:testdoctype1:n=0:doesnotexisthere.html " - "vdsroot/disks/d0/400000000000000.0", - ".*No document with id id:mail:testdoctype1:n=0:doesnotexi.* " - "found.*"); - - // Should test XML with content.. But needs document config for it to work. - // Should be able to create programmatically from testdocman. - ASSERT_MATCH("--toxml --documentconfig '' " - "vdsroot/disks/d0/400000000000000.0", - ".*\n" - "\n" - "overwritten\n" - ".*"); - - // To binary - ASSERT_MATCH("--tobinary --docid id:mail:testdoctype1:n=0:9380.html " - "vdsroot/disks/d0/400000000000000.0", - ".*"); - { - TestDocMan docMan; - document::ByteBuffer buf(output.c_str(), output.size()); - document::Document doc(docMan.getTypeRepo(), buf); - CPPUNIT_ASSERT_EQUAL(std::string( - "\n" - "To be, or not to be: that is the question:\n" - "Whether 'tis nobler in the mind to suffer\n" - "The slings and arrows of outrage\n" - ""), doc.toXml()); - } - - // Fail verification - { - vespalib::LazyFile file("vdsroot/disks/d0/400000000000000.0", 0); - file.write("corrupt", 7, 64); - } - ASSERT_MATCH("-nN vdsroot/disks/d0/400000000000000.0", - ".*lot 0 at timestamp [0-9]+ failed checksum verification.*"); -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/tests/tools/vdsdisktooltest.cpp b/memfilepersistence/src/tests/tools/vdsdisktooltest.cpp deleted file mode 100644 index 1a28cae8d7b..00000000000 --- a/memfilepersistence/src/tests/tools/vdsdisktooltest.cpp +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include -#include -#include - -namespace storage { -namespace memfile { - -struct VdsDiskToolTest : public SingleDiskMemFileTestUtils -{ - framework::defaultimplementation::FakeClock _clock; - - void setUp() override; - void setupRoot(); - - void testSimple(); - - CPPUNIT_TEST_SUITE(VdsDiskToolTest); - CPPUNIT_TEST(testSimple); - CPPUNIT_TEST_SUITE_END(); -}; - -CPPUNIT_TEST_SUITE_REGISTRATION(VdsDiskToolTest); - -#define ASSERT_MATCH(optstring, pattern, exitcode) \ -{ \ - std::ostringstream out; \ - int result = 1; \ - try{ \ - vespalib::AppOptions opts("vdsdisktool " optstring); \ - result = VdsDiskTool::run(opts.getArgCount(), opts.getArguments(), \ - "vdsroot", out, out); \ - } catch (std::exception& e) { \ - out << "Application aborted with exception:\n" << e.what() << "\n"; \ - } \ - CPPUNIT_ASSERT_MATCH_REGEX(pattern, out.str()); \ - CPPUNIT_ASSERT_EQUAL(exitcode, result); \ -} - -namespace { - void createDisk(int i) { - std::ostringstream path; - path << "vdsroot/mycluster/storage/3/disks/d" << i; - CPPUNIT_ASSERT_EQUAL(0, system(("mkdir -p " + path.str()).c_str())); - } -} - -void -VdsDiskToolTest::setUp() -{ - system("rm -rf vdsroot"); -} - -void -VdsDiskToolTest::setupRoot() -{ - system("rm -rf vdsroot"); - createDisk(0); -} - -void -VdsDiskToolTest::testSimple() -{ - // Test syntax page - ASSERT_MATCH("--help", ".*Usage: vdsdisktool .*", 0); - // No VDS installation - ASSERT_MATCH("status", ".*No VDS installations found at all.*", 1); - // Common setup - setupRoot(); - ASSERT_MATCH("status", ".*Disks on storage node 3 in cluster mycluster:\\s*" - "Disk 0: OK\\s*", 0); - // Two disks - system("mkdir -p vdsroot/mycluster/storage/3/disks/d1/"); - ASSERT_MATCH("status", ".*Disks on storage node 3 in cluster mycluster:\\s*" - "Disk 0: OK\\s*" - "Disk 1: OK\\s*", 0); - // Two disks, non-continuous indexes - system("rm -rf vdsroot/mycluster/storage/3/disks/d1/"); - system("mkdir -p vdsroot/mycluster/storage/3/disks/d2/"); - ASSERT_MATCH("status", ".*Disks on storage node 3 in cluster mycluster:\\s*" - "Disk 0: OK\\s*" - "Disk 1: NOT_FOUND - Disk not found during scan.*" - "Disk 2: OK\\s*", 0); - // Status file existing - setupRoot(); - createDisk(1); - MountPointList mountPoints("vdsroot/mycluster/storage/3", - std::vector(), - std::make_unique(std::make_unique(), _clock)); - mountPoints.scanForDisks(); - CPPUNIT_ASSERT_EQUAL(2u, mountPoints.getSize()); - mountPoints[1].addEvent(Device::IO_FAILURE, "Bad", "Found in test"); - mountPoints.writeToFile(); - ASSERT_MATCH("status", ".*Disks on storage node 3 in cluster mycluster:\\s*" - "Disk 0: OK\\s*" - "Disk 1: IO_FAILURE - 0 Bad\\s*", 0); -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/.gitignore b/memfilepersistence/src/vespa/memfilepersistence/.gitignore deleted file mode 100644 index c43cd4d8c3b..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/.depend -/Makefile -/libmemfilepersistence.so.5.1 diff --git a/memfilepersistence/src/vespa/memfilepersistence/CMakeLists.txt b/memfilepersistence/src/vespa/memfilepersistence/CMakeLists.txt deleted file mode 100644 index ab44e62f06f..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(memfilepersistence - SOURCES - $ - $ - $ - $ - $ - $ - $ - INSTALL lib64 - DEPENDS -) diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/.gitignore b/memfilepersistence/src/vespa/memfilepersistence/common/.gitignore deleted file mode 100644 index 7e7c0fe7fae..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.depend -/Makefile diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/CMakeLists.txt b/memfilepersistence/src/vespa/memfilepersistence/common/CMakeLists.txt deleted file mode 100644 index b874aed6242..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(memfilepersistence_common OBJECT - SOURCES - environment.cpp - options.cpp - types.cpp - filespecification.cpp - exceptions.cpp - slotmatcher.cpp - config_lock_guard.cpp - DEPENDS -) diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/config_aliases.h b/memfilepersistence/src/vespa/memfilepersistence/common/config_aliases.h deleted file mode 100644 index 629facd4360..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/config_aliases.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include -#include -#include - -namespace storage { -namespace memfile { - -// Friendly aliases to painfully long config names. -using MemFilePersistenceConfig= vespa::config::storage::StorMemfilepersistenceConfig; -using PersistenceConfig = vespa::config::content::PersistenceConfig; -using DevicesConfig = vespa::config::storage::StorDevicesConfig; - -} // memfile -} // storage - diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/config_lock_guard.cpp b/memfilepersistence/src/vespa/memfilepersistence/common/config_lock_guard.cpp deleted file mode 100644 index a628b538cb1..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/config_lock_guard.cpp +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "config_lock_guard.h" -#include "environment.h" - -namespace storage { -namespace memfile { - -bool -ConfigLockGuardBase::hasPersistenceConfig() const noexcept -{ - return (_env->_persistenceConfig.get() != nullptr); -} - -std::shared_ptr -ConfigLockGuardBase::persistenceConfig() const noexcept -{ - return _env->_persistenceConfig; -} - -bool -ConfigLockGuardBase::hasMemFilePersistenceConfig() const noexcept -{ - return (_env->_config.get() != nullptr); -} - -std::shared_ptr -ConfigLockGuardBase::memFilePersistenceConfig() const noexcept -{ - return _env->_config; -} - -bool -ConfigLockGuardBase::hasDevicesConfig() const noexcept -{ - return (_env->_devicesConfig.get() != nullptr); -} - -std::shared_ptr -ConfigLockGuardBase::devicesConfig() const noexcept -{ - return _env->_devicesConfig; -} - -bool -ConfigLockGuardBase::hasOptions() const noexcept -{ - return (_env->_options.get() != nullptr); -} - -std::shared_ptr -ConfigLockGuardBase::options() const noexcept -{ - return _env->_options; -} - -ConfigWriteLockGuard::ConfigWriteLockGuard(Environment& e) - : ConfigLockGuardBase(e), - _lock(e._configRWLock), - _mutableEnv(&e) -{ -} - -ConfigWriteLockGuard::ConfigWriteLockGuard(ConfigWriteLockGuard&& other) - : ConfigLockGuardBase(std::move(other)), - _lock(other._lock), // Implicit lock stealing, no explicit moving - _mutableEnv(other._mutableEnv) -{ - other._mutableEnv = nullptr; -} - -void -ConfigWriteLockGuard::setPersistenceConfig( - std::unique_ptr cfg) noexcept -{ - mutableEnv()._persistenceConfig = std::move(cfg); -} - -void -ConfigWriteLockGuard::setMemFilePersistenceConfig( - std::unique_ptr cfg) noexcept -{ - mutableEnv()._config = std::move(cfg); -} - -void -ConfigWriteLockGuard::setDevicesConfig( - std::unique_ptr cfg) noexcept -{ - mutableEnv()._devicesConfig = std::move(cfg); -} - -void -ConfigWriteLockGuard::setOptions(std::unique_ptr opts) -{ - mutableEnv()._options = std::move(opts); -} - -ConfigReadLockGuard::ConfigReadLockGuard(const Environment& e) - : ConfigLockGuardBase(e), - _lock(e._configRWLock) -{ -} - -ConfigReadLockGuard::ConfigReadLockGuard(ConfigReadLockGuard&& other) - : ConfigLockGuardBase(std::move(other)), - _lock(other._lock) // Implicit lock stealing, no explicit moving -{ -} - -} // memfile -} // storage - diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/config_lock_guard.h b/memfilepersistence/src/vespa/memfilepersistence/common/config_lock_guard.h deleted file mode 100644 index 72d58120767..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/config_lock_guard.h +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include "config_aliases.h" -#include "options.h" -#include - -namespace storage { -namespace memfile { - -class Environment; - -/** - * Shared guard base allowing read access to existing configs via both - * read and write guard subclasses. - */ -class ConfigLockGuardBase { -public: - explicit ConfigLockGuardBase(const Environment& e) - : _env(&e) - { - } - - ConfigLockGuardBase(ConfigLockGuardBase&& other) - : _env(other._env) - { - // If the source is attempted used after the move, ensure it nukes - // itself with a SIGSEGV. - other._env = nullptr; - } - - // To avoid circular dependencies, all access of Environment internals - // must be in separate .cpp file. - - bool hasPersistenceConfig() const noexcept; - std::shared_ptr persistenceConfig() const noexcept; - - bool hasMemFilePersistenceConfig() const noexcept; - std::shared_ptr - memFilePersistenceConfig() const noexcept; - - bool hasDevicesConfig() const noexcept; - std::shared_ptr devicesConfig() const noexcept; - - bool hasOptions() const noexcept; - std::shared_ptr options() const noexcept; - - ConfigLockGuardBase(const ConfigLockGuardBase&) = delete; - ConfigLockGuardBase& operator=(const ConfigLockGuardBase&) = delete; - -private: - const Environment* _env; -}; - -class ConfigWriteLockGuard : public ConfigLockGuardBase { -public: - explicit ConfigWriteLockGuard(Environment& e); - /** - * Moving a guard transfers ownership of the lock to the move target. It - * is illegal and undefined behavior to attempt to access the environment - * configuration through a guard whose lock has been transferred away. - */ - ConfigWriteLockGuard(ConfigWriteLockGuard&& other); - - // By definition, configs can only be mutated when the writer lock - // is held. - void setPersistenceConfig(std::unique_ptr cfg) noexcept; - void setMemFilePersistenceConfig( - std::unique_ptr cfg) noexcept; - void setDevicesConfig(std::unique_ptr cfg) noexcept; - void setOptions(std::unique_ptr opts); - -private: - vespalib::RWLockWriter _lock; - // This points to the exact same object as the const ref in the base - // and basically serves as an alternative to const_cast. - Environment* _mutableEnv; - - // Hide the fact that we're storing duplicate information to other - // methods. - Environment& mutableEnv() { return *_mutableEnv; } -}; - -class ConfigReadLockGuard : public ConfigLockGuardBase { -public: - explicit ConfigReadLockGuard(const Environment& e); - ConfigReadLockGuard(ConfigReadLockGuard&& other); - - // Config reader methods already implemented in base. - -private: - vespalib::RWLockReader _lock; -}; - - -} // memfile -} // storage - diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/configkeeper.h b/memfilepersistence/src/vespa/memfilepersistence/common/configkeeper.h deleted file mode 100644 index d31bb0500b8..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/configkeeper.h +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class ConfigKeeper - * \class memfile - * - * \brief Utility function for live reconfiguration - * - * When many threads want the same config, we don't want each of these threads - * to subscribe on the same config because of the following reasons: - * - No need to put lots of extra load on the config system. - * - Application doesn't know whether all users have the same config version - * at any given time. - * - * This class implements a utility class for handling this. - */ -#pragma once - -#include - -namespace storage { - -template -class ConfigKeeper { - vespalib::Monitor _configLock; - bool _configUpdated; // Set to true if updating config. - std::unique_ptr _nextConfig; - ConfigClass _config; - -public: - ConfigKeeper() : _configUpdated(false) {} - - void updateConfig(const ConfigClass& config) { - vespalib::MonitorGuard lock(_configLock); - _nextConfig.reset(new ConfigClass(config)); - _configUpdated = true; - } - - void activateNewConfig() { - if (!_configUpdated) return; - vespalib::MonitorGuard lock(_configLock); - _config = *_nextConfig; - _nextConfig.reset(0); - _configUpdated = false; - lock.signal(); - } - - void waitForAnyActivation() { - vespalib::MonitorGuard lock(_configLock); - while (_configUpdated) lock.wait(); - } - - ConfigClass* operator->() { return &_config; } - ConfigClass& operator*() { return _config; } -}; - -} // storage - diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/environment.cpp b/memfilepersistence/src/vespa/memfilepersistence/common/environment.cpp deleted file mode 100644 index 5cbaef2363f..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/environment.cpp +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - -#include "environment.h" -#include -#include -#include -#include -#include - -using config::ConfigGetter; - -namespace storage::memfile { - -namespace { - -template -std::shared_ptr -resolveConfig(const config::ConfigUri& configUri) -{ - return {ConfigGetter::getConfig( - configUri.getConfigId(), configUri.getContext())}; -} - -} - -vespalib::LazyFile::UP -DefaultLazyFileFactory::createFile(const std::string& fileName) const -{ - return vespalib::LazyFile::UP( - new vespalib::LazyFile( - fileName, vespalib::File::DIRECTIO | _flags)); -} - -Environment::Environment(const config::ConfigUri & configUri, - MemFileCache& cache, - MemFileMapper& mapper, - const document::DocumentTypeRepo& typeRepo, - const framework::Clock& clock, - bool ignoreDisks) - : _clock(clock), - _cache(cache), - _memFileMapper(mapper), - _bucketFactory(), - _lazyFileFactory(new DefaultLazyFileFactory( - ignoreDisks ? vespalib::File::READONLY : 0)), - _repo(&typeRepo), - _config(resolveConfig(configUri)), - _persistenceConfig(resolveConfig(configUri)), - _devicesConfig(resolveConfig(configUri)), - _options(std::make_shared(*_config, *_persistenceConfig)) -{ - DeviceManager::UP manager( - new DeviceManager(DeviceMapper::UP(new SimpleDeviceMapper()), - _clock)); - - manager->setPartitionMonitorPolicy( - _devicesConfig->statfsPolicy, _devicesConfig->statfsPeriod); - _mountPoints.reset(new MountPointList(_devicesConfig->rootFolder, - _devicesConfig->diskPath, - std::move(manager))); - - if (!ignoreDisks) { - _mountPoints->init(0); - - // Update full disk setting for partition monitors - for (uint32_t i=0; i<_mountPoints->getSize(); ++i) { - Directory& dir(getDirectory(i)); - if (dir.getPartition().getMonitor() != 0) { - dir.getPartition().getMonitor()->setMaxFillness( - _options->_diskFullFactor); - } - } - } -} - -Types::String -Environment::calculatePathInDir(const Types::BucketId& id, Directory& dir) -{ - vespalib::asciistream os; - os << dir.getPath() << '/'; - // Directories created should only depend on bucket identifier. - document::BucketId::Type seed = id.getId(); - seed = seed ^ (seed >> 32); - vespalib::RandomGen randomizer(static_cast(seed) ^ 0xba5eba11); - - for (uint32_t i = 1; i <= (uint32_t)_config->dirLevels; ++i) { - os << vespalib::make_string( - "%.4x/", - randomizer.nextUint32() % _config->dirSpread); - } - - os << vespalib::make_string("%.8" PRIx64 ".0", id.getId()); - return os.str(); -} - -Environment::~Environment() -{ -} - -Directory& Environment::getDirectory(uint16_t disk) -{ - return (*_mountPoints)[disk]; -} - -void -Environment::addModifiedBucket(const document::BucketId& bid) -{ - vespalib::LockGuard guard(_modifiedBucketsLock); - _modifiedBuckets.push_back(bid); -} - -void -Environment::swapModifiedBuckets(document::BucketId::List & ids) -{ - vespalib::LockGuard guard(_modifiedBucketsLock); - _modifiedBuckets.swap(ids); -} - -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/environment.h b/memfilepersistence/src/vespa/memfilepersistence/common/environment.h deleted file mode 100644 index 75568a707d7..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/environment.h +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::slotfile::MemFileEnvironment - * \ingroup memfile - * - * \brief Keeps environment for MemFile operations - * - * The memfile layer needs quite a lot of stuff set up in order to work. Rather - * than passing all these bits around when creating new slotfiles, we rather - * have an environment where all the static pieces not related to single files - * will be kept. - */ - -#pragma once - -#include "options.h" -#include "types.h" -#include "config_lock_guard.h" -#include "config_aliases.h" -#include -#include -#include -#include - -namespace config { class ConfigUri; } - -namespace storage::memfile { - -class MemFileMapper; -class MemFileCache; - -struct Environment : public Types { - class LazyFileFactory { - public: - virtual ~LazyFileFactory() {}; - - virtual vespalib::LazyFile::UP - createFile(const std::string& fileName) const = 0; - }; - using UP = std::unique_ptr; - - const framework::Clock& _clock; - MemFileCache& _cache; - MemFileMapper& _memFileMapper; - MountPointList::UP _mountPoints; - document::BucketIdFactory _bucketFactory; - std::unique_ptr _lazyFileFactory; - vespalib::Lock _modifiedBucketsLock; - document::BucketId::List _modifiedBuckets; - - Environment(const config::ConfigUri & configUri, - MemFileCache&, - MemFileMapper&, - const document::DocumentTypeRepo&, - const framework::Clock&, - bool ignoreDisks = false); - ~Environment(); - - String calculatePathInDir(const Types::BucketId& id, Directory& dir); - - vespalib::LazyFile::UP createFile(const std::string& fileName) const { - return _lazyFileFactory->createFile(fileName); - } - - Directory& getDirectory(uint16_t disk = 0); - - void addModifiedBucket(const document::BucketId&); - void swapModifiedBuckets(document::BucketId::List &); - - ConfigReadLockGuard acquireConfigReadLock() const { - return ConfigReadLockGuard(*this); - } - - ConfigWriteLockGuard acquireConfigWriteLock() { - return ConfigWriteLockGuard(*this); - } - - /** - * Get the currently assigned document repo in a data race free manner. - * Forms a release/acquire pair with setRepo() - */ - const document::DocumentTypeRepo& repo() const noexcept { - return *_repo.load(std::memory_order_acquire); - } - /** - * Sets the currently assigned document repo in a data race free manner. - * Forms a release/acquire pair with repo() - */ - void setRepo(const document::DocumentTypeRepo* typeRepo) noexcept { - _repo.store(typeRepo, std::memory_order_release); - } -private: - mutable vespalib::RWLock _configRWLock; - /** - * For simplicity, repos are currently kept alive for the duration of the - * process. This means we don't have to care about lifetime management, but - * we still have to ensure writes that set the repo are release/acquired - * paired with their reads. Repos are provided through the SPI and _not_ - * through regular provider-level config subscription, so we therefore do - * not require the config lock to be held when reading/writing. - */ - std::atomic _repo; - /** - * Configs are kept as shared_ptrs to allow lock window to remain as small - * as possible while still retaining thread safety during pointer - * reassignments. - */ - std::shared_ptr _config; - std::shared_ptr _persistenceConfig; - std::shared_ptr _devicesConfig; - /** - * Options is not a true config as per se, but is an aggregate of multiple - * other configs and must thus be protected as if it were. - */ - std::shared_ptr _options; - // We entrust the config guards with access to our internals. - friend class ConfigLockGuardBase; - friend class ConfigWriteLockGuard; - friend class ConfigReadLockGuard; -}; - -struct DefaultLazyFileFactory - : public Environment::LazyFileFactory -{ - int _flags; - - DefaultLazyFileFactory(int flags) : _flags(flags) {} - - vespalib::LazyFile::UP createFile(const std::string& fileName) const override; -}; - -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/exceptions.cpp b/memfilepersistence/src/vespa/memfilepersistence/common/exceptions.cpp deleted file mode 100644 index 05fc40893d8..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/exceptions.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "exceptions.h" - -namespace storage::memfile { - -VESPA_IMPLEMENT_EXCEPTION_SPINE(TimestampExistException); -VESPA_IMPLEMENT_EXCEPTION_SPINE(InconsistentSlotException); -VESPA_IMPLEMENT_EXCEPTION_SPINE(MemFileIoException); -VESPA_IMPLEMENT_EXCEPTION(NoDisksException, vespalib::Exception); - -VESPA_IMPLEMENT_MEMFILE_EXCEPTION(SlotNotFoundException); -VESPA_IMPLEMENT_MEMFILE_EXCEPTION(InvalidArgumentException); -VESPA_IMPLEMENT_MEMFILE_EXCEPTION(InvalidStateException); -VESPA_IMPLEMENT_MEMFILE_EXCEPTION(CorruptMemFileException); -VESPA_IMPLEMENT_MEMFILE_EXCEPTION(MemFileWrapperException); -VESPA_IMPLEMENT_MEMFILE_EXCEPTION(InconsistentException); - -MemFileException::MemFileException(const FileSpecification& file) - : _file(file) -{ } - -MemFileException::~MemFileException() {} - -TimestampExistException::TimestampExistException(const TimestampExistException &) = default; -TimestampExistException::~TimestampExistException() {} - -TimestampExistException::TimestampExistException( - const vespalib::string& message, const FileSpecification& file, - Types::Timestamp ts, const vespalib::string& location, int skipStack) - : Exception(message, location, skipStack + 1), - MemFileException(file), - _timestamp(ts) -{ -} - -InconsistentSlotException::InconsistentSlotException( - const vespalib::string& message, const FileSpecification& file, - const MemSlot& slot, const vespalib::string& location, int skipstack) - : InconsistentException(message, file, location, skipstack + 1), - _slot(slot) -{ -} - -InconsistentSlotException::InconsistentSlotException(const InconsistentSlotException &) = default; -InconsistentSlotException::~InconsistentSlotException() {} - -MemFileIoException::MemFileIoException( - const vespalib::string& msg, const FileSpecification& file, - Type type, const vespalib::string& location, int skipStack) - : IoException(msg, type, location, skipStack + 1), - MemFileException(file) -{ -} - -MemFileIoException::MemFileIoException(const MemFileIoException &) = default; -MemFileIoException::~MemFileIoException() {} - -} // memfile diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/exceptions.h b/memfilepersistence/src/vespa/memfilepersistence/common/exceptions.h deleted file mode 100644 index 6731794578a..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/exceptions.h +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::memfile::IoException - * \ingroup memfile - * - * \brief Exception thrown by memfile layer for IO problems. - * - * Storage needs to know what disk was having issues for disk related problems, - * in case it needs to disable a non-working disk. Some information on what - * file was being operated on while one is having trouble is nice anyhow. Thus - * specific exceptions have been created to keep the file specification of the - * file in question. The MemFile layer may throw some exceptions that aren't - * MemFileExceptions though. These exceptions should not be disk/file related. - */ - -#pragma once - -#include "filespecification.h" -#include -#include - -#define VESPA_DEFINE_MEMFILE_EXCEPTION(name) \ - struct name : public vespalib::Exception, public MemFileException { \ - name(const vespalib::string& message, const FileSpecification& file, \ - const vespalib::string& location, int skipStack = 0); \ - name(const name &); \ - name & operator = (const name &); \ - name(name &&) = default; \ - name & operator = (name &&) = default; \ - ~name(); \ - VESPA_DEFINE_EXCEPTION_SPINE(name); \ -}; - -#define VESPA_IMPLEMENT_MEMFILE_EXCEPTION(name) \ - name::name(const vespalib::string& message, const FileSpecification& file, \ - const vespalib::string& location, int skipStack) \ - : vespalib::Exception(message, location, skipStack + 1), \ - MemFileException(file) {} \ - name::name(const name &) = default; \ - name & name::operator = (const name &) = default; \ - name::~name() {} \ - VESPA_IMPLEMENT_EXCEPTION_SPINE(name); - -namespace storage::memfile { - -VESPA_DEFINE_EXCEPTION(NoDisksException, vespalib::Exception); - -/** - * \class storage::memfile::MemFileException - * \ingroup memfile - * - * \brief Interface to implement for exceptions that contain a file specification specifying what memfile was problematic. - */ -class MemFileException : protected Types { - FileSpecification _file; - -public: - MemFileException(const FileSpecification&); - virtual ~MemFileException() = 0; - - const FileSpecification& getFile() const { return _file; } -}; - -VESPA_DEFINE_MEMFILE_EXCEPTION(SlotNotFoundException); -VESPA_DEFINE_MEMFILE_EXCEPTION(InvalidArgumentException); -VESPA_DEFINE_MEMFILE_EXCEPTION(InvalidStateException); -VESPA_DEFINE_MEMFILE_EXCEPTION(CorruptMemFileException); -VESPA_DEFINE_MEMFILE_EXCEPTION(MemFileWrapperException); - -/** - * \class storage::InconsistentException - * \ingroup memfile - * - * \brief Thrown by MemFile::verifyConsistent() if inconsistent - */ -VESPA_DEFINE_MEMFILE_EXCEPTION(InconsistentException); - -/** - * @class storage::TimestampExistException - * @ingroup filestorage - * - * @brief Thrown by SlotFile::write() when timestamp given is already in use. - */ -class TimestampExistException : public vespalib::Exception, - public MemFileException -{ - Timestamp _timestamp; -public: - TimestampExistException(const vespalib::string& message, - const FileSpecification&, Timestamp ts, - const vespalib::string& location, int skipstack = 0); - TimestampExistException(const TimestampExistException &); - ~TimestampExistException(); - - VESPA_DEFINE_EXCEPTION_SPINE(TimestampExistException); - - Timestamp getTimestamp() const { return _timestamp; } -}; - -/** - * @class storage::InconsistentSlotException - * @ingroup filestorage - * - * @brief Thrown by MemFile::verifyConsistent() if a slot is inconsistent - */ -class InconsistentSlotException : public InconsistentException { - MemSlot _slot; - -public: - InconsistentSlotException(const vespalib::string& message, - const FileSpecification&, const MemSlot& slot, - const vespalib::string& location, int skipstack = 0); - InconsistentSlotException(const InconsistentSlotException &); - ~InconsistentSlotException(); - - VESPA_DEFINE_EXCEPTION_SPINE(InconsistentSlotException); -}; - -class MemFileIoException : public vespalib::IoException, - public MemFileException -{ -public: - MemFileIoException(const vespalib::string& msg, const FileSpecification&, - Type type, const vespalib::string& location, - int skipStack = 0); - MemFileIoException(const MemFileIoException &); - ~MemFileIoException(); - - VESPA_DEFINE_EXCEPTION_SPINE(MemFileIoException); -}; - -} // memfile diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/filespecification.cpp b/memfilepersistence/src/vespa/memfilepersistence/common/filespecification.cpp deleted file mode 100644 index 09a3205dec3..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/filespecification.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - -#include "filespecification.h" -#include - -namespace storage { -namespace memfile { - -FileSpecification::FileSpecification(const BucketId& bucket, Directory& dir, - const String& path) - : _bucketId(bucket), - _dir(&dir), - _path(path), - _wantedVersion(TRADITIONAL_SLOTFILE) -{ - if (dir.getState() != Device::OK) { - throw vespalib::IllegalStateException( - "Attempt to create file specification for file on disk that " - "is not available: " + dir.toString(), VESPA_STRLOC); - } -} - -void -FileSpecification::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "FileSpecification(" << _bucketId << ", " << *_dir << ", " << _path - << ", wanted version 0x" << std::hex << _wantedVersion << std::dec - << ")"; -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/filespecification.h b/memfilepersistence/src/vespa/memfilepersistence/common/filespecification.h deleted file mode 100644 index e7898fad288..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/filespecification.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::slotfile::FileSpecification - * \ingroup memfile - * - * \brief Information about the file currently worked on. - * - * The file specification specifies what file a given MemFile should work on. - */ - -#pragma once - -#include "types.h" -#include -#include - -namespace storage { -namespace memfile { - -class MemFileEnvironment; - -class FileSpecification : private Types, - public vespalib::Printable -{ - BucketId _bucketId; - Directory* _dir; - String _path; - FileVersion _wantedVersion; - -public: - FileSpecification(const BucketId&, Directory&, const String& path); - - void setWantedVersion(FileVersion v) { _wantedVersion = v; } - - const document::BucketId& getBucketId() const { return _bucketId; } - Directory& getDirectory() const { return *_dir; } - const String& getPath() const { return _path; } - FileVersion getWantedFileVersion() const { return _wantedVersion; } - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - - bool operator==(const FileSpecification& o) const { - return (_bucketId == o._bucketId && _dir == o._dir - && _path == o._path && _wantedVersion == o._wantedVersion); - } -}; - -} // storage -} // memfile diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp b/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp deleted file mode 100644 index 34b2f23ed65..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/options.cpp +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "options.h" -#include -#include - -#include -LOG_SETUP(".persistence.slotfile.options"); - -namespace storage::memfile { - -Options::Options() - : _minimumFileMetaSlots(512), - _maximumFileMetaSlots(0), - _minimumFileHeaderBlockSize(102848), - _maximumFileHeaderBlockSize(0), - _minimumFileSize(1048576), - _maximumFileSize(0), - _fileBlockSize(4096), - _revertTimePeriod(300 * 1000000ull), - _keepRemoveTimePeriod(604800 * 1000000ull), - _maxDocumentVersions(5), - _cacheSize(0), - _initialIndexRead(65536), - _maximumGapToReadThrough(65536), - _diskFullFactor(0.98), - _growFactor(2.0), - _overrepresentMetaDataFactor(1.2), - _overrepresentHeaderBlockFactor(1.1), - _defaultRemoveDocType() -{ -} - -Options::Options(const Options &) noexcept = default; -Options & Options::operator=(const Options &) = default; -Options::~Options() { } - -Options::Options(const vespa::config::storage::StorMemfilepersistenceConfig& newConfig, - const vespa::config::content::PersistenceConfig& newPersistenceConfig) - : _minimumFileMetaSlots(newConfig.minimumFileMetaSlots), - _maximumFileMetaSlots(newConfig.maximumFileMetaSlots), - _minimumFileHeaderBlockSize(newConfig.minimumFileHeaderBlockSize), - _maximumFileHeaderBlockSize(newConfig.maximumFileHeaderBlockSize), - _minimumFileSize(newConfig.minimumFileSize), - _maximumFileSize(newConfig.maximumFileSize), - _fileBlockSize(newConfig.fileBlockSize), - _revertTimePeriod(newPersistenceConfig.revertTimePeriod * 1000000ll), - _keepRemoveTimePeriod( - newPersistenceConfig.keepRemoveTimePeriod * 1000000ll), - _maxDocumentVersions( - newPersistenceConfig.maximumVersionsOfSingleDocumentStored), - _cacheSize(newConfig.cacheSize), - _initialIndexRead(newConfig.initialIndexRead), - _maximumGapToReadThrough(newConfig.maximumGapToReadThrough), - _diskFullFactor(newConfig.diskFullFactor), - _growFactor(newConfig.growFactor), - _overrepresentMetaDataFactor(newConfig.overrepresentMetaDataFactor), - _overrepresentHeaderBlockFactor(newConfig.overrepresentHeaderBlockFactor), - _defaultRemoveDocType( - newConfig.store50BackwardsCompatibleRemoveEntriesWithDoctype) -{ - validate(); -} - -bool -Options::operator==(const Options& options) const { - if (_minimumFileMetaSlots == options._minimumFileMetaSlots - && _maximumFileMetaSlots == options._maximumFileMetaSlots - && _minimumFileHeaderBlockSize - == options._minimumFileHeaderBlockSize - && _maximumFileHeaderBlockSize - == options._maximumFileHeaderBlockSize - && _minimumFileSize == options._minimumFileSize - && _maximumFileSize == options._maximumFileSize - && _fileBlockSize == options._fileBlockSize - && _revertTimePeriod == options._revertTimePeriod - && _maxDocumentVersions == options._maxDocumentVersions - && _keepRemoveTimePeriod == options._keepRemoveTimePeriod - && _cacheSize == options._cacheSize - && _initialIndexRead == options._initialIndexRead - && _maximumGapToReadThrough == options._maximumGapToReadThrough - && _diskFullFactor == options._diskFullFactor - && _defaultRemoveDocType == options._defaultRemoveDocType) - { - return true; - } - return false; -} - -namespace { - template - void verifyAligned(Number n, uint32_t alignSize, const char* name) { - if (n % alignSize != 0) { - std::ostringstream ost; - ost << name << " " << n - << " must be dividable by block alignment size " << alignSize; - throw vespalib::IllegalStateException( - ost.str(), VESPA_STRLOC); - } - } -} - -void Options::validate() -{ - uint32_t tmp32 = 0; - - // REVERT / KEEP REMOVE TIME PERIODS - if (_revertTimePeriod > _keepRemoveTimePeriod) { - LOG(warning, "Keep all time period (%" PRIu64 ") is set larger than keep " - "removes time period (%" PRIu64 ". Adjusting keep removes " - "period to match", - _revertTimePeriod.getTime(), _keepRemoveTimePeriod.getTime()); - _keepRemoveTimePeriod = _revertTimePeriod; - } - if (_maxDocumentVersions < 1) { - LOG(warning, "Max number of document versions attempted set to 0. " - "This is a bad idea for all the obvious reasons. Forcing " - "used value to be 1."); - _maxDocumentVersions = 1; - } - // MINIMUM FILE SIZES - if (_minimumFileMetaSlots < 1) { - LOG(warning, "Minimum file meta slots is not allowed to be less than " - "1. Setting it to 1."); - _minimumFileMetaSlots = 1; - } - if (_minimumFileMetaSlots > 1024*1024) { - LOG(warning, "Minimum file meta slots is not allowed to be more than " - "%u. Setting it to %u.", 1024*1024, 1024*1024); - _minimumFileMetaSlots = 1024*1024; - } - if (_minimumFileHeaderBlockSize > 2*1024*1024*1024u) { - LOG(warning, "Minimum file header block size is not allowed to be above" - " 2 GB. Altering it from %u B to 2 GB.", - _minimumFileHeaderBlockSize); - _minimumFileHeaderBlockSize = 2*1024*1024*1024u; - } - if (_minimumFileSize % _fileBlockSize != 0) { - tmp32 = _fileBlockSize - * ((_minimumFileSize + _fileBlockSize - 1) / _fileBlockSize); - LOG(warning, "Min file size %u not a multiplum of file block size %u. " - "Increasing minimum filesize to %u to match.", - _minimumFileSize, _fileBlockSize, tmp32); - _minimumFileSize = tmp32; - } - // MAXIMUM FILE SIZES - if (_maximumFileMetaSlots != 0 - && _maximumFileMetaSlots < _minimumFileMetaSlots) - { - LOG(warning, "Maximum file meta slots cannot be less than the minimum. " - "Adjusting it from %u to %u.", - _maximumFileMetaSlots, _minimumFileMetaSlots); - _maximumFileMetaSlots = _minimumFileMetaSlots; - } - if (_maximumFileHeaderBlockSize != 0 - && _maximumFileHeaderBlockSize < _minimumFileHeaderBlockSize) - { - LOG(warning, "Maximum file header block size cannot be less than the " - "minimum. Adjusting it from %u to %u.", - _maximumFileHeaderBlockSize, _minimumFileHeaderBlockSize); - _maximumFileHeaderBlockSize = _minimumFileHeaderBlockSize; - } - if (_maximumFileSize != 0 && _maximumFileSize < _minimumFileSize) { - LOG(warning, "Maximum file size cannot be less than the " - "minimum. Adjusting it from %u to %u.", - _maximumFileSize, _minimumFileSize); - _maximumFileSize = _minimumFileSize; - } - if (_maximumFileSize % _fileBlockSize != 0) { - tmp32 = _fileBlockSize - * ((_maximumFileSize + _fileBlockSize - 1) / _fileBlockSize); - LOG(warning, "Max file size %u not a multiplum of file block size %u. " - "Increasing maximum to %u bytes to match.", - _maximumFileSize, _fileBlockSize, tmp32); - _maximumFileSize = tmp32; - } - - if (_growFactor < 1.0 || _growFactor >= 100.0) { - throw vespalib::IllegalStateException( - "The grow factor needs to be in the range [1, 100].", - VESPA_STRLOC); - } - - if (!_defaultRemoveDocType.empty()) { - // Log the usage of this option to make it visible, as it is not - // something most people should use. - LOG(info, - "Will write remove entries in 5.0 backwards compatible mode. By " - "default this will be done using the '%s' document type unless " - "the document identifier specifies otherwise.", - _defaultRemoveDocType.c_str()); - } -} - -void Options::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; - std::string s("\n" + indent + " "); - - out << "SlotFile options:" - << s << "Minimum file meta slots: " << _minimumFileMetaSlots - << s << "Maximum file meta slots: " << _maximumFileMetaSlots - << s << "Minimum file header block size: " - << _minimumFileHeaderBlockSize << " b" - << s << "Maximum file header block size: " - << _maximumFileHeaderBlockSize << " b" - << s << "Minimum file size: " << _minimumFileSize << " b" - << s << "Maximum file size: " << _maximumFileSize << " b" - << s << "Filesystem block size: " << _fileBlockSize << " b" - << s << "Revert time period: " << _revertTimePeriod << " microsecs" - << s << "Keep remove time period: " - << _keepRemoveTimePeriod << "microsecs" - << s << "Max document versions: " << _maxDocumentVersions - << s << "Cache size: " << _cacheSize - << s << "Initial index read: " << _initialIndexRead << " b" - << s << "Maximum gap to read through: " - << _maximumGapToReadThrough << " b" - << s << "Disk full factor: " << _diskFullFactor - << s << "Grow factor: " << _growFactor - << s << "Overrepresent meta data factor: " - << _overrepresentMetaDataFactor - << s << "Overrepresent header block factor: " - << _overrepresentHeaderBlockFactor - << s << "Write removes with blank documents of default type: " - << _defaultRemoveDocType - << ""; -} - -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/options.h b/memfilepersistence/src/vespa/memfilepersistence/common/options.h deleted file mode 100644 index a7310c236a1..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/options.h +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::Options - * @ingroup filestorage - * - * @brief Options used by slotfiles - * - * To avoid the need for static variables which cannot be altered while the - * system is running, and which forces all slotfile instances to work with the - * same options, this options class has been created to contain all the options - * a slotfile will use. - * - * @author H�kon Humberset - * @date 2005-10-26 - */ - -#pragma once - -#include -#include -#include -#include -#include - -namespace storage::memfile { - -struct Options : public vespalib::Printable -{ - // Parameters from def file. See config file for comments. - - // FILE SIZE PARAMETERS - - uint32_t _minimumFileMetaSlots; - uint32_t _maximumFileMetaSlots; - uint32_t _minimumFileHeaderBlockSize; - uint32_t _maximumFileHeaderBlockSize; - uint32_t _minimumFileSize; - uint32_t _maximumFileSize; - uint32_t _fileBlockSize; - - // CONSISTENCY PARAMETERS - framework::MicroSecTime _revertTimePeriod; - framework::MicroSecTime _keepRemoveTimePeriod; - uint32_t _maxDocumentVersions; - - // PERFORMANCE PARAMETERS - uint64_t _cacheSize; - uint32_t _initialIndexRead; - uint32_t _maximumGapToReadThrough; - - double _diskFullFactor; - double _growFactor; - double _overrepresentMetaDataFactor; - double _overrepresentHeaderBlockFactor; - - // COMPATIBILITY PARAMETERS - // If non-empty, will cause remove entries to be written with a blank - // document containing only the document type and identifier rather than - // just writing a document id with no document at all. Note that if a - // document identifier contains a type string it will override this default - // value. - // This is a feature for backwards compatibility with 5.0, as it chokes - // when trying to read remove entries without a document. - vespalib::string _defaultRemoveDocType; - - /** - * Creates a new slotfile options instance. Implemented in header file, - * such that the current defaults can be easily viewed. - */ - Options(); - Options(const Options &) noexcept; - Options & operator=(const Options &); - ~Options(); - - Options(const vespa::config::storage::StorMemfilepersistenceConfig& newConfig, - const vespa::config::content::PersistenceConfig& newPersistenceConfig); - - void validate() const { const_cast(*this).validate(); } - void validate(); - - /** Printable implementation */ - void print(std::ostream& out, bool verbose, - const std::string& indent) const override; - - bool operator==(const Options& options) const; -}; - -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/slotmatcher.cpp b/memfilepersistence/src/vespa/memfilepersistence/common/slotmatcher.cpp deleted file mode 100644 index 702f7388c48..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/slotmatcher.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "slotmatcher.h" -#include - -namespace storage { -namespace memfile { - -Types::Timestamp -SlotMatcher::Slot::getTimestamp() const -{ - return _slot.getTimestamp(); -} - -bool -SlotMatcher::Slot::isRemove() const -{ - return _slot.deleted(); -} - -const document::GlobalId& -SlotMatcher::Slot::getGlobalId() const -{ - return _slot.getGlobalId(); -} - -document::Document::UP -SlotMatcher::Slot::getDocument(bool headerOnly) const -{ - return _file.getDocument(_slot, headerOnly ? HEADER_ONLY : ALL); -} - -document::DocumentId -SlotMatcher::Slot::getDocumentId() const -{ - return _file.getDocumentId(_slot); -} - -} // memfile -} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/slotmatcher.h b/memfilepersistence/src/vespa/memfilepersistence/common/slotmatcher.h deleted file mode 100644 index cb93dd678e8..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/slotmatcher.h +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::slotfile::SlotMatcher - * \ingroup memfile - * - * \brief Implement this to create a filter for MemSlot instances. - * - * Many operations want to do something to a subset of the slots in a file. - * Such operations can retrieve the slots that matches using an implementation - * of this filter. - * - * Creating a slot matcher, you should give information of what type of data - * you want to preload from disk. Typically you want to preload entries you - * need such as to prevent many disk accesses, but if there is some data you - * only need for a few entries, you can use the functions supplied in the - * matcher to get these instances even though they are not cached for all - * entries. - */ - -#pragma once - -#include - -namespace storage { -namespace memfile { - -class MemFile; - -class SlotMatcher : private Types { -public: - enum PreloadFlag { - PRELOAD_META_DATA_ONLY = 0x0, - PRELOAD_BODY = 0x1, - PRELOAD_HEADER = 0x3, - PRELOAD_DOC_ID = 0x7 - }; - -protected: - SlotMatcher(PreloadFlag preld) : _preload(preld) {} - - PreloadFlag _preload; - -public: - class Slot { - private: - const MemSlot& _slot; - const MemFile& _file; - - public: - Slot(const MemSlot& slot, const MemFile& file) - : _slot(slot), - _file(file) {}; - - /** - Returns the timestamp of the slot. - */ - Timestamp getTimestamp() const; - - /** - * Returns whether a slot is a remove, either regular - * or unrevertable. - */ - bool isRemove() const; - - /** - Returns the global id of the slot. - */ - const GlobalId& getGlobalId() const; - - /** - * Get the document, optionally just the header. If not preloaded, will load - * this document from disk. - */ - Document::UP getDocument(bool headerOnly) const; - - document::DocumentId getDocumentId() const; - }; - - virtual ~SlotMatcher() {} - - virtual bool match(const Slot&) = 0; - - /** Do what is needed to preload wanted content. */ - void preload(MemFile&) const {}; -}; - -} // storage -} // memfile - diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp b/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp deleted file mode 100644 index 19898674555..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/types.cpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "types.h" -#include -#include -#include - -namespace storage::memfile { - -const framework::MicroSecTime Types::MAX_TIMESTAMP(framework::MicroSecTime::max()); -const framework::MicroSecTime Types::UNSET_TIMESTAMP(0); - -void -Types::verifyLegalFlags(uint32_t flags, uint32_t legal, const char* operation) -{ - if ((flags & legal) != flags) { - std::ostringstream ost; - ost << "Invalid flags given to operation " << operation << ". " - << std::hex << flags << " given, but only " << legal - << " are legal."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); - } -} - -std::ostream& -operator<<(std::ostream& os, const DataLocation& loc) -{ - os << "DataLocation(" - << std::dec - << loc._pos - << ", " - << loc._size - << ")"; - return os; -} - -const char* -Types::getMemFileFlagName(MemFileFlag flag) { - switch (flag) { - case FILE_EXIST: return "FILE_EXIST"; - case HEADER_BLOCK_READ: return "HEADER_BLOCK_READ"; - case BODY_BLOCK_READ: return "BODY_BLOCK_READ"; - case BUCKET_INFO_OUTDATED: return "BUCKET_INFO_OUTDATED"; - case SLOTS_ALTERED: return "SLOTS_ALTERED"; - case LEGAL_MEMFILE_FLAGS: assert(false); // Not a single flag - default: return "INVALID"; - } -} - -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/common/types.h b/memfilepersistence/src/vespa/memfilepersistence/common/types.h deleted file mode 100644 index 09485b972a6..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/common/types.h +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::slotfile::Types - * \ingroup memfile - * - * \brief This class defines and includes some types used in the slotfile layer. - * - * As many of the types are used many places in the layer, we define them here - * rather than in one random class using them. This also makes it easy to switch - * implementation by switching out which class to use here. - * - * This class should not have any members, virtual classes or anything. We don't - * want it to add to the memory footprint of classes, as it will be used also - * by classes kept many times in memory cache. - */ -#pragma once - - -#include -#include -#include - -namespace storage::memfile { - -/** - * \class storage::slotfile::DataLocation - * \ingroup memfile - * - * \brief Points to data in a file storing documents. - * - * This file stores info on where header and body parts of document are stored. - * It is really format specific data, but for now it is implemented globally. - * - * All unused locations should be size zero pointing to address zero. A size - * of zero with a non-zero position is invalid, and used to indicate that this - * value is not set yet. (Typically when data isn't persisted to disk yet) - */ -struct DataLocation { - uint32_t _pos; - uint32_t _size; - - DataLocation() : _pos(1), _size(0) {} // pos 1 size 0 is invalid value. - DataLocation(uint32_t pos, uint32_t sz) : _pos(pos), _size(sz) {} - - uint32_t size() const { return _size; } - - uint32_t endPos() const { return _pos + _size; } - - bool valid() const { return (_size > 0 || _pos == 0); } - - bool operator==(const DataLocation& other) const { - return (_pos == other._pos && _size == other._size); - } - bool operator!=(const DataLocation& other) const { - return ! (*this == other); - } - - bool operator<(const DataLocation& other) const { - if (_pos == other._pos) { - return _size < other._size; - } - - return _pos < other._pos; - } - - bool contains(const DataLocation& other) const { - return (_pos <= other._pos && _pos + _size >= other._pos + other._size); - } -}; - -std::ostream& operator<<(std::ostream&, const DataLocation&); - -struct Types { - typedef document::BucketId BucketId; - typedef document::Document Document; - typedef document::DocumentId DocumentId; - typedef document::GlobalId GlobalId; - typedef framework::MicroSecTime Timestamp; - typedef Timestamp RevertToken; - typedef vespalib::string String; - typedef spi::BucketInfo BucketInfo; - - static const framework::MicroSecTime MAX_TIMESTAMP; - static const framework::MicroSecTime UNSET_TIMESTAMP; - - enum FileVersion { - UNKNOWN = 0, - TRADITIONAL_SLOTFILE = 0xABCD0001 - }; - - enum SlotFlag { - IN_USE = 0x01, - DELETED = 0x02, - DELETED_IN_PLACE = 0x04, - LEGAL_PERSISTED_SLOT_FLAGS = 0x07, - - // States not stored in file. As we have set aside 16 bits for the - // flags in the fileformat, but use so few, we use some of the - // unused bits in the memory representation to store memory state. - ALTERED_IN_MEMORY = 0x02 << 8, - CHECKSUM_OUTDATED = 0x04 << 8, - - // Masks to check for multiple bits - UNUSED = 0xf8f8 - }; - - enum GetFlag { - ALL = 0, - HEADER_ONLY = 0x1, - LEGAL_GET_FLAGS = 0x1 - }; - - enum IteratorFlag { - ITERATE_GID_UNIQUE = 0x1, - ITERATE_REMOVED = 0x2, - LEGAL_ITERATOR_FLAGS = 0x3 - }; - - enum DocContentFlag { - HAS_HEADER_ONLY, - HAS_BODY - }; - - enum DocumentPart { - HEADER, - BODY - }; - - enum MemFileFlag { - FILE_EXIST = 0x0001, - HEADER_BLOCK_READ = 0x0002, - BODY_BLOCK_READ = 0x0004, - BUCKET_INFO_OUTDATED = 0x0008, - SLOTS_ALTERED = 0x0010, - LEGAL_MEMFILE_FLAGS = 0x001f - }; - - enum FileVerifyFlags { - DONT_VERIFY_HEADER = 0x0001, - DONT_VERIFY_BODY = 0x0002, - LEGAL_VERIFY_FLAGS = 0x0003 - }; - - enum FlushFlag { - NONE = 0, - CHECK_NON_DIRTY_FILE_FOR_SPACE = 1 - }; - - enum GetLocationsFlag { - NON_PERSISTED_LOCATIONS = 0x0001, - PERSISTED_LOCATIONS = 0x0002, - NO_SLOT_LIST = 0x0004 - }; - - enum DocumentCopyType { - DEEP_COPY, - SHALLOW_COPY - }; - - static const char* getDocumentPartName(DocumentPart part) { - switch (part) { - case HEADER: return "Header"; - case BODY: return "Body"; - default: return "Invalid"; - } - } - - static const char* getFileVersionName(FileVersion version) { - switch (version) { - case UNKNOWN: return "UNKNOWN"; - case TRADITIONAL_SLOTFILE: return "TRADITIONAL_SLOTFILE"; - default: return "INVALID"; - } - } - - static const char* getMemFileFlagName(MemFileFlag flag); - - static void verifyLegalFlags(uint32_t flags, uint32_t legal, const char* operation); -protected: - ~Types() {} // Noone should refer to objects as Types objects -}; - -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/.gitignore b/memfilepersistence/src/vespa/memfilepersistence/device/.gitignore deleted file mode 100644 index 7e7c0fe7fae..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.depend -/Makefile diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/CMakeLists.txt b/memfilepersistence/src/vespa/memfilepersistence/device/CMakeLists.txt deleted file mode 100644 index 88bb6513ba2..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(memfilepersistence_device OBJECT - SOURCES - device.cpp - disk.cpp - partition.cpp - directory.cpp - devicemapper.cpp - devicemanager.cpp - ioevent.cpp - partitionmonitor.cpp - mountpointlist.cpp - DEPENDS -) diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/device.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/device.cpp deleted file mode 100644 index cd1fcafb52c..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/device.cpp +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "device.h" -#include "ioevent.h" -#include -#include - -namespace storage::memfile { - -Device::Device(DeviceManager& manager) - : _manager(manager) -{} - -Device::~Device() {} - -std::string Device::getStateString(State s) -{ - switch (s) { - case OK: return "OK"; - case TOO_MANY_OPEN_FILES: return "TOO_MANY_OPEN_FILES"; - case NOT_FOUND: return "NOT_FOUND"; - case PATH_FAILURE: return "PATH_FAILURE"; - case NO_PERMISSION: return "NO_PERMISSION"; - case IO_FAILURE: return "IO_FAILURE"; - case INTERNAL_FAILURE: return "INTERNAL_FAILURE"; - case DISABLED_BY_ADMIN: return "DISABLED_BY_ADMIN"; - default: - { - std::ostringstream ost; - ost << "UNKNOWN(" << s << ")"; - return ost.str(); - } - } -} - -void -Device::print(std::ostream& out, bool, const std::string&) const -{ - const IOEvent* event = getLastEvent(); - if (event == 0) { - out << Device::OK; - } else { - out << event->getState() << " "; - out << event->getTimestamp() << " "; - std::string desc = event->getDescription(); - std::replace(desc.begin(), desc.end(), '\n', ' '); - out << desc; - } -} - -void -Device::clearEvents() -{ - _events.clear(); -} - -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/device.h b/memfilepersistence/src/vespa/memfilepersistence/device/device.h deleted file mode 100644 index 63413e61b0a..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/device.h +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::Device - * \ingroup persistence - * - * @brief Class holding information about a device. - * - * Base class for devices, such as directories, partitions and disks. - */ - -#pragma once - -#include -#include - -namespace storage { - -namespace memfile { - -class IOEvent; -class DeviceManager; - -class Device : public vespalib::Printable { -private: - // These objects are not possible to copy. They represents physical - // resources on a computer - Device(const Device&); - Device& operator=(Device&); - -protected: - DeviceManager& _manager; - std::list _events; - - Device(DeviceManager& manager); - -public: - /** - * Storage device states. Most serious states are at the bottom of the - * list. If a single state is requested from the device, the one with - * the highest value wins through. - */ - enum State { - OK, - NOT_FOUND, // Not found - PATH_FAILURE, // Illegal path - NO_PERMISSION, // Permission problems - INTERNAL_FAILURE, // Probably problem with process. - IO_FAILURE, // Disk problems - TOO_MANY_OPEN_FILES, // Too many open files so we can't use disk. - // This is a global problem that will not be stored - // as disk state, but must exist in order to be - // able to report event. - DISABLED_BY_ADMIN // If disabled through admin tool - }; - - static std::string getStateString(State s); - - virtual ~Device(); - - virtual void addEvent(const IOEvent& e) = 0; - virtual void clearEvents(); - virtual const IOEvent* getLastEvent() const = 0; - - const std::list& getEvents() const { return _events; } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - -}; - -} // memfile - -} // storage - diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp deleted file mode 100644 index bf03b798e1c..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.cpp +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "devicemanager.h" -#include -#include - -namespace storage::memfile { - -DeviceManager::DeviceManager(DeviceMapper::UP mapper, - const framework::Clock& clock) - : _deviceMapper(std::move(mapper)), - _disks(), - _partitions(), - _directories(), - _eventListeners(), - _statPolicy(vespa::config::storage::StorDevicesConfig::STAT_DYNAMIC), - _statPeriod(0), - _clock(clock) -{ -} - -DeviceManager::~DeviceManager() {} - -void -DeviceManager::setPartitionMonitorPolicy( - vespa::config::storage::StorDevicesConfig::StatfsPolicy policy, uint32_t period) -{ - _statPolicy = policy; - _statPeriod = period; - for (std::map::iterator it - = _partitions.begin(); it != _partitions.end(); ++it) - { - Partition& p(*it->second); - if (p.getMonitor() != 0) p.getMonitor()->setPolicy(policy, period); - } -} - -void DeviceManager::notifyDiskEvent(Disk& d, const IOEvent& e) -{ - for (std::set::iterator it = _eventListeners.begin(); - it != _eventListeners.end(); ++it) - { - assert(*it != 0); - (*it)->handleDiskEvent(d, e); - } -} - -void -DeviceManager::notifyDirectoryEvent(Directory& dir, const IOEvent& e) -{ - for (std::set::iterator it = _eventListeners.begin(); - it != _eventListeners.end(); ++it) - { - assert(*it != 0); - (*it)->handleDirectoryEvent(dir, e); - } -} - -void -DeviceManager::notifyPartitionEvent(Partition& part, const IOEvent& e) -{ - for (std::set::iterator it = _eventListeners.begin(); - it != _eventListeners.end(); ++it) - { - assert(*it != 0); - (*it)->handlePartitionEvent(part, e); - } -} - -void -DeviceManager::addIOEventListener(IOEventListener& listener) -{ - _eventListeners.insert(&listener); -} - -void -DeviceManager::removeIOEventListener(IOEventListener& listener) -{ - _eventListeners.erase(&listener); -} - -Directory::SP -DeviceManager::getDirectory(const std::string& dir, uint16_t index) -{ - std::map::iterator it = - _directories.find(dir); - if (it != _directories.end()) { - return it->second; - } - Directory::SP d(new Directory(*this, index, dir)); - _directories[dir] = d; - return d; -} - -Directory::SP -DeviceManager::deserializeDirectory(const std::string& serialized) -{ - // Deserialize object - Directory::SP d(new Directory(serialized, *this)); - // If not existing, just add it. - std::map::iterator it = - _directories.find(d->getPath()); - if (it == _directories.end()) { - _directories[d->getPath()] = d; - return d; - } - // If already existing, merge info with existing entry. - it->second->addEvents(*d); - return it->second; -} - -Partition::SP -DeviceManager::getPartition(const std::string& path) -{ - try{ - std::string mountPoint(_deviceMapper->getMountPoint(path)); - uint64_t id = _deviceMapper->getPartitionId(mountPoint); - std::map::iterator it( - _partitions.find(mountPoint)); - if (it != _partitions.end()) { - return it->second; - } - Partition::SP part(new Partition(*this, id, mountPoint)); - if (part->getMonitor() != 0) { - part->getMonitor()->setPolicy(_statPolicy, _statPeriod); - } - _partitions[mountPoint] = part; - return part; - } catch (vespalib::IoException& e) { - // If we fail to create partition, due to having IO troubles getting - // partition id or mount point, create a partition that doesn't - // correspond to a physical device containing the error found. - Partition::SP part(new Partition(*this, -1, path)); - part->addEvent(IOEvent::createEventFromIoException( - e, - _clock.getTimeInSeconds().getTime())); - _partitions[path] = part; - return part; - } -} - -Disk::SP -DeviceManager::getDisk(const std::string& path) -{ - try{ - int devnr = _deviceMapper->getDeviceId(path); - std::map::iterator it = _disks.find(devnr); - if (it != _disks.end()) { - return it->second; - } - Disk::SP disk(new Disk(*this, devnr)); - _disks[devnr] = disk; - return disk; - } catch (vespalib::IoException& e) { - // Use negative ints for illegal ids. Make sure they don't already - // exist - int devnr = -1; - while (_disks.find(devnr) != _disks.end()) --devnr; - // If we fail to create partition, due to having IO troubles getting - // partition id or mount point, create a partition that doesn't - // correspond to a physical device containing the error found. - Disk::SP disk(new Disk(*this, devnr)); - disk->addEvent(IOEvent::createEventFromIoException( - e, - _clock.getTimeInSeconds().getTime())); - _disks[devnr] = disk; - return disk; - } -} - -void -DeviceManager::printXml(vespalib::XmlOutputStream& xos) const -{ - using namespace vespalib::xml; - xos << XmlTag("devicemanager"); - xos << XmlTag("mapper") << XmlAttribute("type", _deviceMapper->getName()) - << XmlEndTag(); - xos << XmlTag("devices"); - for (std::map::const_iterator diskIt = _disks.begin(); - diskIt != _disks.end(); ++diskIt) - { - xos << XmlTag("disk") << XmlAttribute("deviceId", diskIt->first); - for (std::map::const_iterator partIt - = _partitions.begin(); partIt != _partitions.end(); ++partIt) - { - if (partIt->second->getDisk() != *diskIt->second) continue; - xos << XmlTag("partition") - << XmlAttribute("id", partIt->second->getId()) - << XmlAttribute("mountpoint", partIt->second->getMountPoint()); - if (partIt->second->getMonitor() != 0) { - xos << *partIt->second->getMonitor(); - } - for (std::map::const_iterator dirIt - = _directories.begin(); dirIt != _directories.end(); - ++dirIt) - { - if (dirIt->second->getPartition() != *partIt->second) continue; - xos << XmlTag("directory") - << XmlAttribute("index", dirIt->second->getIndex()) - << XmlAttribute("path", dirIt->second->getPath()) - << XmlEndTag(); - } - xos << XmlEndTag(); - } - xos << XmlEndTag(); - } - xos << XmlEndTag() << XmlEndTag(); -} - -} // memfile diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.h b/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.h deleted file mode 100644 index b8bbb735769..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/devicemanager.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::DeviceManager - * \ingroup persistence - * - * \brief Class keeping information about all devices. - * - * This class keeps track of all the devices so they can be looked up. - */ -#pragma once - -#include "devicemapper.h" -#include "directory.h" -#include "disk.h" -#include "ioevent.h" -#include "partition.h" -#include -#include -#include - -namespace storage::memfile { - -class DeviceManager : public vespalib::XmlSerializable { - using StatfsPolicy = vespa::config::storage::StorDevicesConfig::StatfsPolicy; - DeviceMapper::UP _deviceMapper; - std::map _disks; - std::map _partitions; - std::map _directories; - std::set _eventListeners; - StatfsPolicy _statPolicy; - uint32_t _statPeriod; - const framework::Clock& _clock; - - void setFindDeviceFunction(); -public: - using UP = std::unique_ptr; - - DeviceManager(DeviceMapper::UP mapper, const framework::Clock& clock); - DeviceManager(const DeviceManager&) = delete; - DeviceManager& operator=(const DeviceManager&) = delete; - ~DeviceManager(); - - void setPartitionMonitorPolicy(StatfsPolicy, uint32_t period = 0); - - void notifyDiskEvent(Disk& disk, const IOEvent& e); - void notifyDirectoryEvent(Directory& dir, const IOEvent& e); - void notifyPartitionEvent(Partition& part, const IOEvent& e); - - void addIOEventListener(IOEventListener& listener); - void removeIOEventListener(IOEventListener& listener); - - Directory::SP getDirectory(const std::string& dir, uint16_t index); - Directory::SP deserializeDirectory(const std::string& serialized); - Partition::SP getPartition(const std::string& path); - Disk::SP getDisk(const std::string& path); - - std::vector getDirectories(const Disk& disk) const; - std::vector getDirectories(const Partition& part) const; - - StatfsPolicy getStatPolicy() const { return _statPolicy; } - uint32_t getStatPeriod() const { return _statPeriod; } - - void printXml(vespalib::XmlOutputStream&) const override; - - const framework::Clock& getClock() const { return _clock; } -}; - -} // memfile diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp deleted file mode 100644 index 53dd4375d8a..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.cpp +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "devicemapper.h" -#include -#include -#include -#include -#include - -#include -LOG_SETUP(".persistence.devicemapper"); - -namespace storage::memfile { - -namespace { - uint64_t getDevice(const std::string& path) { - struct stat info; - if (stat(path.c_str(), &info) != 0) { - std::ostringstream ost; - ost << "Failed to run stat to find data on file " << path - << ": errno(" << errno << ") - " << vespalib::getLastErrorString() << "."; - throw vespalib::IoException( - ost.str(), vespalib::IoException::getErrorType(errno), - VESPA_STRLOC); - } - return info.st_dev; - } -} - -AdvancedDeviceMapper::AdvancedDeviceMapper() - : _mountPoints() -{ - // Initialize the mount point map - std::ifstream is; - is.exceptions(std::ifstream::badbit); // Throw exception on failure - is.open("/proc/mounts"); - init(is); -} - -void -AdvancedDeviceMapper::init(std::istream& is) -{ - std::string line; - while (std::getline(is, line)) { - vespalib::StringTokenizer st(line, " \t\f\r\n", ""); - if (st[0] == "none") { - LOG(debug, "Ignoring special mount point '%s'.", line.c_str()); - continue; - } - if (st.size() < 3 || st[1][0] != '/') { - LOG(warning, "Found unexpected line in /proc/mounts: '%s'.", - line.c_str()); - continue; - } - std::string mountPoint(st[1]); - try{ - uint64_t deviceId = getDevice(mountPoint); - LOG(debug, "Added mountpoint '%s' with device id %" PRIu64 ".", - mountPoint.c_str(), deviceId); - _mountPoints[deviceId] = mountPoint; - } catch (vespalib::Exception& e) { - LOG(info, "Failed to get device of mountpoint %s. This is normal " - "for some special mountpoints, and doesn't matter unless " - "the device is used by VDS: %s", - mountPoint.c_str(), e.getMessage().c_str()); - } - } -} - -std::string -AdvancedDeviceMapper::getMountPoint(const std::string& fileOnFS) const -{ - uint64_t dev = getDevice(fileOnFS); - std::map::const_iterator it(_mountPoints.find(dev)); - if (it == _mountPoints.end()) { - std::ostringstream ost; - ost << "Failed to find a device for file '" << fileOnFS << "'. Stat " - << "returned device " << dev << " but only the following devices " - << "are known:"; - for (it = _mountPoints.begin(); it != _mountPoints.end(); ++it) { - ost << " (" << it->first << " - " << it->second << ")"; - } - throw vespalib::IoException( - ost.str(), vespalib::IoException::INTERNAL_FAILURE, - VESPA_STRLOC); - } - return it->second; -} - -uint64_t -AdvancedDeviceMapper::getPartitionId(const std::string& fileOnFS) const -{ - return getDevice(fileOnFS); -} - -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.h b/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.h deleted file mode 100644 index e8fcef9ff8a..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/devicemapper.h +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * The device mapper is used to get some interesting information for - * storage devies. - */ -#pragma once - -#include -#include -#include -#include - -namespace storage { - -namespace memfile { - -/** - * @class DeviceMapper - * @ingroup persistence - * - * @brief Maps directories to partition and disk information. - */ -struct DeviceMapper { - typedef std::unique_ptr UP; - - virtual ~DeviceMapper() {} - - virtual const char* getName() const = 0; - - virtual std::string getMountPoint(const std::string& fileOnFS) const = 0; - virtual uint64_t getPartitionId(const std::string& fileOnFS) const = 0; - virtual uint64_t getDeviceId(const std::string& fileOnFS) const = 0; -}; - -/** - * @class SimpleDeviceMapper - * @ingroup persistence - * - * @brief Simple device mapper, not trying to detect any information. - * - * This simple device mapper, assumes all directories used are actually - * mountpoints, and that all mountpoints are on separate disks. This returns - * dummy device numbers. - * - * Using this, each directory used will be handled separately, and there is no - * dependency on information to retrieve from OS. - */ -class SimpleDeviceMapper : public DeviceMapper { - mutable std::map _devices; - mutable int _lastDevice; - - SimpleDeviceMapper(const SimpleDeviceMapper&); - SimpleDeviceMapper& operator=(const SimpleDeviceMapper&); - -public: - SimpleDeviceMapper() : _devices(), _lastDevice(0) {} - - uint64_t getPartitionId(const std::string& fileOnFS) const override { - std::map::const_iterator it = _devices.find(fileOnFS); - if (it != _devices.end()) { - return it->second; - } - int dev = ++_lastDevice; - _devices[fileOnFS] = dev; - return dev; - } - std::string getMountPoint(const std::string& path) const override { return path; } - uint64_t getDeviceId(const std::string& fileOnFS) const override { - return getPartitionId(fileOnFS); - } - const char* getName() const override { return "Simple (All directories on individual fake devices)"; } -}; - -/** - * @class AdvancedDeviceMapper - * @ingroup persistence - * - * @brief Device mapper trying to find a real physical model using stat/statfs. - * - * Using this device mapper, stat/statfs will be used to try to find a real - * model. Directories mapping to common components wil cause all directories to - * fail if the common component fails. - */ -struct AdvancedDeviceMapper : public DeviceMapper { - std::map _mountPoints; - - AdvancedDeviceMapper(); - void init(std::istream&); - - std::string getMountPoint(const std::string& fileOnFS) const override; - uint64_t getPartitionId(const std::string& fileOnFS) const override; - uint64_t getDeviceId(const std::string& fileOnFS) const override { - // Not found a way to detect partitions on common device. - // Returning partition ids for now. - return getPartitionId(fileOnFS); - } - const char* getName() const override { return "Advanced (Read devices attempted found)"; } -}; - -} - -} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/directory.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/directory.cpp deleted file mode 100644 index ce5ddaf6e2d..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/directory.cpp +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "directory.h" -#include "devicemanager.h" -#include - -#include -LOG_SETUP(".persistence.device.directory"); - -namespace storage { - -namespace memfile { - -const IOEvent* -Directory::getLastEvent() const -{ - if (!_events.empty()) return &_events.back(); - return _partition->getLastEvent(); -} - -Device::State -Directory::getState() const -{ - const IOEvent* event = getLastEvent(); - return (event ? event->getState() : Device::OK); -} - -void -Directory::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - out << _path << " "; - Device::print(out, verbose, indent); -} - -Directory::Directory(DeviceManager& manager, uint16_t index, - const std::string& path) - : Device(manager), - _index(index), - _path(path), - _partition(manager.getPartition(path)) -{ - assert(_partition.get()); -} - -namespace { - struct Entry { - std::string path; - Device::State status; - std::string description; - Entry(); - ~Entry(); - }; - - Entry::Entry() {} - Entry::~Entry() {} - - Entry parseDirectoryString(const std::string& serialized) { - while (1) { - Entry e; - std::string::size_type pos1 = serialized.find(' '); - if (pos1 == std::string::npos) break; - e.path = serialized.substr(0, pos1); - std::string::size_type pos2 = serialized.find(' ', pos1 + 1); - std::string num = serialized.substr(pos1 + 1, pos2 - pos1 - 1); - char* c; - e.status = static_cast( - strtoul(num.c_str(), &c, 10)); - if (*c != '\0') break; - if (pos2 != std::string::npos) { - e.description = serialized.substr(pos2 + 1); - } - return e; - } - std::string msg = "Illegal line in disk status file: '" + serialized - + "'. Ignoring it."; - LOG(warning, "%s", msg.c_str()); - throw vespalib::IllegalArgumentException(msg, VESPA_STRLOC); - } -} - -Directory::Directory(const std::string& serialized, - DeviceManager& manager) - : Device(manager), - _index(0), - _path(parseDirectoryString(serialized).path), - _partition(manager.getPartition(_path)) -{ - assert(_partition.get()); - Entry e = parseDirectoryString(serialized); - if (e.status != Device::OK) { - addEvent(IOEvent(manager.getClock().getTimeInSeconds().getTime(), - e.status, e.description, VESPA_STRLOC)); - } -} - -void Directory::addEvent(const IOEvent& e) -{ - switch (e.getState()) { - case Device::IO_FAILURE: - _partition->addEvent(e); - break; - case Device::PATH_FAILURE: - case Device::NO_PERMISSION: - case Device::INTERNAL_FAILURE: - case Device::DISABLED_BY_ADMIN: - default: - if (!e.isGlobal()) { - _events.push_back(e); - } - _manager.notifyDirectoryEvent(*this, e); - } -} - -void -Directory::addEvent(Device::State s, - const std::string& description, - const std::string& location) -{ - addEvent(IOEvent( - _manager.getClock().getTimeInSeconds().getTime(), - s, - description, - location)); - -} - -void Directory::addEvents(const Directory& d) -{ - std::list events; - events.insert(events.end(), d.getEvents().begin(), d.getEvents().end()); - events.insert(events.end(), d.getPartition().getEvents().begin(), - d.getPartition().getEvents().end()); - events.insert(events.end(), d.getPartition().getDisk().getEvents().begin(), - d.getPartition().getDisk().getEvents().end()); - for (std::list::const_iterator it = events.begin(); - it != events.end(); ++it) - { - addEvent(*it); - } -} - -} // memfile - -} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/directory.h b/memfilepersistence/src/vespa/memfilepersistence/device/directory.h deleted file mode 100644 index c7df91f4a48..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/directory.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::Directory - * \ingroup persistence - * - * \brief Class representing a directory used by Vespa storage. - * - * IMPORTANT: Directory objects may be generated for faulty directories too, - * thus creating the object must not result in a disk operation. - */ -#pragma once - -#include "partition.h" - -namespace storage { - -namespace memfile { - -class Directory : public Device { - uint16_t _index; - std::string _path; - Partition::SP _partition; - - // Only DeviceManager can create these objects, so we only need - // to cope with these constructors being so similar there. - Directory(DeviceManager&, uint16_t index, const std::string& path); - Directory(const std::string& serialized, DeviceManager& manager); - - void addEvents(const Directory& d); - - friend class DeviceManager; - -public: - using SP = std::shared_ptr; - void setIndex(uint16_t index) { _index = index; } // Used when deserializing - - uint16_t getIndex() const { return _index; } - const std::string& getPath() const { return _path; } - Partition& getPartition() { return *_partition; } - const Partition& getPartition() const { return *_partition; } - - const IOEvent* getLastEvent() const override; - void addEvent(const IOEvent& e) override; - virtual void addEvent(Device::State s, - const std::string& description, - const std::string& location); - - State getState() const; - bool isOk() const { return (getLastEvent() == 0); } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - bool operator==(const Directory& d) const { return (_path == d._path); } - bool operator!=(const Directory& d) const { return (_path != d._path); } - - // Easy access functions, using the partition monitor to query state of - // partition - - /** Query whether partition is full after adding given amount of data. */ - bool isFull(int64_t afterAdding = 0, double maxFillRate = -1) const { - return _partition->getMonitor() == 0 - || _partition->getMonitor()->isFull(afterAdding, maxFillRate); - } - -}; - -} // memfile - -} // storage - diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/disk.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/disk.cpp deleted file mode 100644 index bc30098a877..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/disk.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "disk.h" -#include "devicemanager.h" - -namespace storage { - -namespace memfile { - -Disk::Disk(DeviceManager& manager, uint64_t id) - : Device(manager), - _id(id) -{ -} - -void Disk::addEvent(const IOEvent& e) -{ - if (!e.isGlobal()) { - _events.push_back(e); - } - _manager.notifyDiskEvent(*this, e); -} - -const IOEvent* -Disk::getLastEvent() const -{ - if (getEvents().size() > 0) - return &getEvents().back(); - return 0; -} - -void -Disk::print(std::ostream& out, bool verbose, const std::string& indent) const -{ - out << "Disk id: " << _id << " "; - Device::print(out, verbose, indent); -} - -} // memfile - -} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/disk.h b/memfilepersistence/src/vespa/memfilepersistence/device/disk.h deleted file mode 100644 index 3e15d31d93f..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/disk.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::Disk - * \ingroup persistence - * - * \brief Class representing a storage unit on a node. - * - * Class representing a storage unit on a node, which can be a physical disk, or - * a device set up by a RAID controller or similar. - * - * IMPORTANT: Disk objects may be generated for faulty disks too, thus creating - * the object must not result in a disk operation. - */ - -#pragma once - -#include "device.h" - -namespace storage { - -namespace memfile { - -class Disk : public Device { - uint64_t _id; - - Disk(DeviceManager&, uint64_t id); - - friend class DeviceManager; - -public: - using SP = std::shared_ptr; - - uint64_t getId() const { return _id; } - - void addEvent(const IOEvent& e) override; - const IOEvent* getLastEvent() const override; - - bool operator==(const Disk& disk) const { return (_id == disk._id); } - bool operator!=(const Disk& disk) const { return (_id != disk._id); } - void print(std::ostream& out, bool verbose, const std::string& indent) const override; -}; - -} // memfile - -} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/ioevent.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/ioevent.cpp deleted file mode 100644 index ac1edea6a93..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/ioevent.cpp +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "ioevent.h" -#include -#include - -namespace storage { - -namespace memfile { - -IOEvent::IOEvent() - : _state(Device::OK), - _description(), - _location(), - _global(false), - _timestamp(0) -{} - -IOEvent::IOEvent(const IOEvent &) = default; -IOEvent & IOEvent::operator = (const IOEvent &) = default; - -IOEvent::~IOEvent() {} - -namespace { - vespalib::string stripBacktrace(const vespalib::string& s) { - vespalib::string::size_type pos = s.find("Backtrace:"); - if (pos == vespalib::string::npos) return s; - while (pos > 0 && (s[pos - 1] == ' ' || s[pos - 1] == '\n')) { - --pos; - } - return s.substr(0, pos); - } -} - -IOEvent::IOEvent(uint32_t timestamp, Device::State s, const vespalib::string& description, - const vespalib::string& location, bool global) - : _state(s), - _description(stripBacktrace(description)), - _location(location), - _global(global), - _timestamp(timestamp) -{ -} - -IOEvent -IOEvent::createEventFromErrno(uint32_t timestamp, - int error, const vespalib::string& extraInfo, - const vespalib::string& location) -{ - vespalib::string err(vespalib::getErrorString(error)); - err += ": " + extraInfo; - switch (error) { - case ENOENT: - return IOEvent(timestamp, Device::NOT_FOUND, err, location); - case ENOTDIR: - case ENAMETOOLONG: - case ELOOP: - case EISDIR: // Using directory as file - case EOPNOTSUPP: // Operation not supported by filesystem - case EROFS: - case EMLINK: - case ENXIO: - case ESPIPE: // Descriptor is a pip/socket/fifo - return IOEvent(timestamp, Device::PATH_FAILURE, err, location); - case EACCES: - return IOEvent(timestamp, Device::NO_PERMISSION, err, location); - case EIO: // IO error occured. - case EINTR: // Read from slow device interrupted before any data. - return IOEvent(timestamp, Device::IO_FAILURE, err, location); - case EMFILE: - return IOEvent(timestamp, Device::TOO_MANY_OPEN_FILES, err, - location, true); - case EAGAIN: // Non-blocking read but no data available - case EBADF: // Invalid file descriptor - case EFAULT: // Buffer pointer invalid - case EINVAL: // Faulty input parameter - case ENFILE: - default: - return IOEvent(timestamp, Device::INTERNAL_FAILURE, err, location); - } -} - -IOEvent -IOEvent::createEventFromIoException(vespalib::IoException& e, uint32_t timestamp) -{ - Device::State type = Device::INTERNAL_FAILURE; - switch (e.getType()) { - case vespalib::IoException::NOT_FOUND: - type = Device::NOT_FOUND; break; - case vespalib::IoException::ILLEGAL_PATH: - type = Device::PATH_FAILURE; break; - case vespalib::IoException::NO_PERMISSION: - type = Device::NO_PERMISSION; break; - case vespalib::IoException::DISK_PROBLEM: - type = Device::IO_FAILURE; break; - case vespalib::IoException::TOO_MANY_OPEN_FILES: - type = Device::TOO_MANY_OPEN_FILES; break; - case vespalib::IoException::INTERNAL_FAILURE: - case vespalib::IoException::NO_SPACE: - case vespalib::IoException::CORRUPT_DATA: - case vespalib::IoException::DIRECTORY_HAVE_CONTENT: - case vespalib::IoException::FILE_FULL: - case vespalib::IoException::ALREADY_EXISTS: - case vespalib::IoException::UNSPECIFIED: - type = Device::INTERNAL_FAILURE; break; - } - return IOEvent(timestamp, type, e.getMessage(), e.getLocation()); -} - -void -IOEvent::print(std::ostream & os, bool verbose, const std::string& indent) const -{ - (void) indent; - os << "IOEvent("; - os << Device::getStateString(_state); - if (verbose) { - if (_description.size() > 0) { - os << ", " << _description; - } - if (_location.size() > 0) { - os << ", " << _location; - } - os << ", time " << _timestamp; - } - os << ")"; -} - -} // memfile - -} // storage diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/ioevent.h b/memfilepersistence/src/vespa/memfilepersistence/device/ioevent.h deleted file mode 100644 index c22c49e084c..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/ioevent.h +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::IOEvent - * \ingroup persistence - * - * \brief Class representing an IO event. An event tied to a device. - */ -#pragma once - -#include "device.h" - -namespace vespalib { class IoException; } -namespace storage { - -namespace memfile { - -class IOEvent : public vespalib::Printable { -public: - IOEvent(); - - IOEvent(uint32_t timestamp, - Device::State s, - const vespalib::string & description, - const vespalib::string & location, - bool global = false); - IOEvent(const IOEvent &); - IOEvent & operator = (const IOEvent &); - IOEvent(IOEvent &&) = default; - IOEvent & operator = (IOEvent &&) = default; - - ~IOEvent(); - - static IOEvent createEventFromErrno(uint32_t timestamp, - int error, - const vespalib::string& extraInfo = "", - const vespalib::string& location = ""); - static IOEvent createEventFromIoException(vespalib::IoException& e, - uint32_t timestamp); - - Device::State getState() const { return _state; } - const vespalib::string& getDescription() const { return _description; } - - void print(std::ostream& out, bool verbose, - const std::string& indent) const override; - - /** - * Global events aren't tied to device they was found in. They should not - * be saved on each device or be a reason to disable one. - */ - bool isGlobal() const { return _global; } - - uint32_t getTimestamp() const { return _timestamp; } - -private: - Device::State _state; - vespalib::string _description; - vespalib::string _location; - bool _global; - uint32_t _timestamp; -}; - -class Directory; -class Partition; -class Disk; - -/** - * \class storage::IOEventListener - * \ingroup persistence - * - * \brief Interface to implement if you want IO events. Register at manager. - */ -struct IOEventListener { - virtual void handleDirectoryEvent(Directory& dir, const IOEvent& e) = 0; - virtual void handlePartitionEvent(Partition& part, const IOEvent& e) = 0; - virtual void handleDiskEvent(Disk& disk, const IOEvent& e) = 0; - - virtual ~IOEventListener() {} -}; - -} - -} - diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp b/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp deleted file mode 100644 index 5e1d835e3a2..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.cpp +++ /dev/null @@ -1,643 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "mountpointlist.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -LOG_SETUP(".persistence.mountpointlist"); - -namespace storage::memfile { - -using vespalib::getLastErrorString; -using vespalib::DirPointer; - -MountPointList::MountPointList(const std::string& vdsRoot, - const std::vector& diskPath, - DeviceManager::UP manager) - : framework::XmlStatusReporter("mountpointlist", "Disk directories"), - _deviceManager(std::move(manager)), - _vdsRoot(vdsRoot), - _diskPath(diskPath), - _mountPoints(0) -{ -} - -MountPointList::~MountPointList() {} - -spi::PartitionStateList -MountPointList::getPartitionStates() const -{ - spi::PartitionStateList list(_mountPoints.size()); - for (uint32_t i=0; i<_mountPoints.size(); ++i) { - if (!(_mountPoints[i]->isOk())) { - const IOEvent* event = _mountPoints[i]->getLastEvent(); - list[i] = spi::PartitionState(spi::PartitionState::DOWN, event->getDescription()); - } - } - - return list; -} - -void -MountPointList::init(uint16_t diskCount) -{ - initDisks(); - scanForDisks(); - readFromFile(); - if (verifyHealthyDisks(diskCount == 0 ? -1 : diskCount)) { - // Initialize monitors after having initialized disks, such as to - // not create them for invalid disks. - initializePartitionMonitors(); - } - if (diskCount != 0 && _mountPoints.size() != diskCount) { - std::ostringstream ost; - ost << _mountPoints.size() - << " mount points found. Expected " << diskCount - << " mount points to exist."; - LOG(error, "%s", ost.str().c_str()); - throw config::InvalidConfigException(ost.str(), VESPA_STRLOC); - } -} - -void -MountPointList::initDisks() -{ - if (_diskPath.empty()) return; - - using vespalib::make_string; - - vespalib::string vdsDisksPath = make_string("%s/disks", _vdsRoot.c_str()); - vespalib::mkdir(vdsDisksPath); - - for (size_t diskIndex = 0; diskIndex < _diskPath.size(); ++diskIndex) { - auto disk_path = make_string( - "%s/d%zu", vdsDisksPath.c_str(), diskIndex); - if (pathExists(disk_path)) continue; - - vespalib::mkdir(_diskPath[diskIndex]); - - try { - vespalib::symlink(_diskPath[diskIndex], disk_path); - } catch (vespalib::IoException& dummy) { - // The above mkdir() created disk_path as a directory, or a - // subdirectory of disk_path, which is OK. - (void) dummy; - } - } -} - -void -MountPointList::initializePartitionMonitors() -{ - std::set seen; - for (uint32_t i=0; i<_mountPoints.size(); ++i) { - if (!(_mountPoints[i]->isOk())) continue; - Partition* part = &_mountPoints[i]->getPartition(); - std::set::const_iterator it(seen.find(part)); - if (it == seen.end()) { - part->initializeMonitor(); - seen.insert(part); - } - } -} - -void -MountPointList::scanForDisks() -{ - _mountPoints.clear(); - std::vector entries; - DirPointer dir(opendir((_vdsRoot + "/disks").c_str())); - struct dirent* entry; - if (dir) while ((entry = readdir(dir))) { - if (entry == 0) { - std::ostringstream ost; - ost << "Failed to read directory \"" << _vdsRoot << "/disks\", " - << "errno " << errno << ": " << getLastErrorString(); - throw vespalib::IoException(ost.str(), - vespalib::IoException::DISK_PROBLEM, VESPA_STRLOC); - } - std::string name(reinterpret_cast(&entry->d_name)); - assert(name.size() > 0); - if (name[0] == '.') continue; - // To be a valid d name, size must be at least 2 - if (name.size() < 2 || name[0] != 'd') { - LOG(warning, "File %s in disks directory is faulty named for a " - "disk directory, ignoring it.", name.c_str()); - continue; - } - char* endp; - uint32_t diskNr = strtoul(name.c_str()+1, &endp, 10); - // If rest of name is not a number, ignore - if (*endp != '\0') { - LOG(warning, "File %s in disks directory is faulty named for a " - "disk directory, ignoring it.", name.c_str()); - continue; - } - // If number is out of range, ignore.. - if (diskNr >= 254) { - LOG(warning, "Ignoring disk directory %s, as max directories have " - "been set to 254.", name.c_str()); - continue; - } - - // Valid disk directory.. Add entry.. - if (entries.size() <= diskNr) { - entries.resize(diskNr + 1); - } - LOG(debug, "Found disk directory %u: %s", diskNr, name.c_str()); - entries[diskNr] = _deviceManager->getDirectory( - _vdsRoot + "/disks/" + name, diskNr); - - // We only care about directories (or symlinks). DT_UNKNOWN must be handled explicitly. - if (entry->d_type != DT_DIR && entry->d_type != DT_LNK && entry->d_type != DT_UNKNOWN) { - std::ostringstream ost; - ost << "File " << name << " in disks directory is not a directory."; - LOG(warning, "%s", ost.str().c_str()); - entries[diskNr]->addEvent(Device::PATH_FAILURE, - ost.str(), VESPA_STRLOC); - } - - // Not all filesystems support d_type. Have to stat if this equals DT_UNKNOWN. - if (entry->d_type == DT_UNKNOWN) { - struct stat st; - lstat(entries[diskNr]->getPath().c_str(), &st); - if (!S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) { - std::ostringstream ost; - ost << "File " << name << " in disks directory is not a directory."; - LOG(warning, "%s", ost.str().c_str()); - entries[diskNr]->addEvent(Device::PATH_FAILURE, - ost.str(), VESPA_STRLOC); - } - } - } else if (errno == ENOENT) { - std::ostringstream ost; - ost << "Disk directory \"" << _vdsRoot << "/disks\" not created. VDS " - << "needs this to know which disks to use. See vespa doc."; - throw NoDisksException(ost.str(), VESPA_STRLOC); - } else { - std::ostringstream ost; - ost << "Failed to open directory \"" << _vdsRoot << "/disks\", errno " - << errno << ": " << getLastErrorString(); - throw vespalib::IoException(ost.str(), - vespalib::IoException::DISK_PROBLEM, VESPA_STRLOC); - } - // Assign found disks to the instance - _mountPoints.resize(entries.size()); - for (uint32_t i=0; i<_mountPoints.size(); ++i) { - if (!entries[i].get()) { - if (!_mountPoints[i].get() || - _mountPoints[i]->getState() == Device::OK) - { - std::ostringstream ost; - ost << _vdsRoot + "/disks/d" << i; - _mountPoints[i] = _deviceManager->getDirectory(ost.str(), i); - _mountPoints[i]->addEvent( - Device::NOT_FOUND, - "Disk not found during scanning of disks directory", - VESPA_STRLOC); - } - LOG(warning, "Disk %u was not found.", i); - } else if (!_mountPoints[i].get() || - _mountPoints[i]->getState() == Device::NOT_FOUND) - { - _mountPoints[i] = entries[i]; - } - } -} - -namespace { - /** - * Get the disk nr of the given mountpoint, - * or -1 if the mountpoint is illegal. - */ - int getDiskNr(const std::string& mountPoint) { - std::string::size_type pos1 = mountPoint.rfind('/'); - if (pos1 == std::string::npos || - pos1 + 2 >= mountPoint.size() || - mountPoint[pos1+1] != 'd') - { - return -1; - } - char* endp; - std::string digit(mountPoint.substr(pos1+2)); - const char* digitptr = digit.c_str(); - int diskNr = strtoul(digitptr, &endp, 10); - if (digitptr[0] == '\0' || *endp != '\0') return -1; - return diskNr; - } -} - -void -MountPointList::readFromFile() -{ - std::vector entries; - // Read entries from disk - std::ifstream is; - // Throw exception if failing to read file - is.exceptions(std::ifstream::badbit); - is.open(getDiskStatusFileName().c_str()); - std::string line("EOF"); - while (std::getline(is, line)) { - if (line == "EOF") { break; } - Directory::SP dir = _deviceManager->deserializeDirectory(line); - int diskNr = getDiskNr(dir->getPath()); - if (diskNr == -1) { - LOG(warning, "Found illegal disk entry '%s' in vds disk file %s.", - line.c_str(), getDiskStatusFileName().c_str()); - } else { - dir->setIndex(diskNr); - if (entries.size() <= static_cast(diskNr)) { - entries.resize(diskNr + 1); - } - entries[diskNr] = dir; - } - } - if (line != "EOF" || std::getline(is, line)) { - LOG(warning, "Disk status file %s did not end in EOF.", - getDiskStatusFileName().c_str()); - } - // Assign entries to this instance - if (_mountPoints.size() < entries.size()) { - _mountPoints.resize(entries.size()); - } - for (uint32_t i=0; igetState() != Device::OK && - entries[i]->getState() != Device::NOT_FOUND) - { - _mountPoints[i] = entries[i]; - } - } -} - -void -MountPointList::writeToFile() const -{ - try{ - std::string filename(getDiskStatusFileName()); - std::string tmpFilename(filename + ".tmp"); - std::ofstream os(tmpFilename.c_str()); - if (os.fail()) { - LOG(warning, "Failed to open %s.tmp for writing. Not writing " - "disks.status file.", filename.c_str()); - return; - } - for (std::vector::const_iterator it - = _mountPoints.begin(); it != _mountPoints.end(); ++it) - { - if (it->get() && - (*it)->getState() != Device::OK) - { - os << **it << "\n"; - } - } - os << "EOF"; - os.close(); - if (os.fail()) { - LOG(warning, "Failed to write %s.tmp. disks.status file might now " - "be corrupt as we failed while writing it.", - filename.c_str()); - return; - } - vespalib::rename(tmpFilename, filename, false, false); - LOG(debug, "Mount point list saved to file %s.", filename.c_str()); - } catch (std::exception& e) { - LOG(warning, "Failed to write disk status file: %s", e.what()); - } -} - -namespace { - void testMountPoint(Directory& mountPoint) { - struct stat filestats; - if (stat(mountPoint.getPath().c_str(), &filestats) != 0) { - switch (errno) { - case ENOTDIR: - case ENAMETOOLONG: - case ENOENT: - case EACCES: - case ELOOP: - { - mountPoint.addEvent(Device::PATH_FAILURE, - getLastErrorString(), - VESPA_STRLOC); - return; - } - case EIO: - { - mountPoint.addEvent(Device::IO_FAILURE, - getLastErrorString(), VESPA_STRLOC); - return; - } - case EFAULT: - default: - assert(0); // Should never happen - } - } - // At this point we know the mount point exists.. - if (!(S_ISDIR(filestats.st_mode))) { - mountPoint.addEvent( - Device::PATH_FAILURE, - "The path exist, but is not a directory.", - VESPA_STRLOC); - } - } - - struct Chunk { - uint32_t nr; - uint32_t total; - - Chunk() : nr(0), total(0) {} // Invalid - bool valid() const { return (nr < total); } - }; - - Chunk getChunkDef(const std::string& mountPoint) { - vespalib::File file(mountPoint + "/chunkinfo"); - file.open(vespalib::File::READONLY); - std::string buffer; - buffer.resize(200, '\0'); - size_t read(file.read(&buffer[0], buffer.size(), 0)); - buffer.resize(read); - vespalib::StringTokenizer tokenizer(buffer, "\n", ""); - - Chunk chunk; - if (tokenizer.size() < 3) { - return chunk; - } - - char *c; - chunk.nr = strtoul(tokenizer[1].c_str(), &c, 10); - if (tokenizer[1].c_str() + tokenizer[1].size() != c) return Chunk(); - chunk.total = strtoul(tokenizer[2].c_str(), &c, 10); - if (tokenizer[2].c_str() + tokenizer[2].size() != c) return Chunk(); - return chunk; - } - - void writeChunkDef(Chunk c, const std::string& mountPoint) { - vespalib::File file(mountPoint + "/chunkinfo"); - file.open(vespalib::File::CREATE | vespalib::File::TRUNC, true); - std::ostringstream ost; - ost << "# This file tells VDS what data this mountpoint may contain.\n" - << c.nr << "\n" - << c.total << "\n"; - std::string content(ost.str()); - file.write(&content[0], content.size(), 0); - } - - Device::State getDeviceState(vespalib::IoException::Type type) { - using vespalib::IoException; - switch (type) { - case IoException::ILLEGAL_PATH: return Device::PATH_FAILURE; - case IoException::NO_PERMISSION: return Device::NO_PERMISSION; - case IoException::DISK_PROBLEM: return Device::IO_FAILURE; - case IoException::INTERNAL_FAILURE: return Device::INTERNAL_FAILURE; - default: ; - } - return Device::OK; - } - - bool emptyDir(Directory& dir) { - const std::string& path(dir.getPath()); - errno = 0; - DirPointer dirdesc(opendir(path.c_str())); - struct dirent* entry; - if (dirdesc) while ((entry = readdir(dirdesc))) { - if (errno) break; - std::string name(reinterpret_cast(&entry->d_name)); - if (name == "." || name == "..") continue; - return false; - } - if (dirdesc == 0 || errno) { - std::ostringstream ost; - ost << "Failed to read directory \"" << path << "\", " - << "errno " << errno << ": " << getLastErrorString(); - dir.addEvent(getDeviceState(vespalib::IoException::getErrorType(errno)), - ost.str(), - VESPA_STRLOC); - throw vespalib::IoException(ost.str(), - vespalib::IoException::DISK_PROBLEM, VESPA_STRLOC); - } - return true; - } - - struct WriteStatusFileIfFailing { - MountPointList& _list; - bool _failed; - - WriteStatusFileIfFailing(MountPointList& list) - : _list(list), _failed(false) {} - ~WriteStatusFileIfFailing() { - if (_failed) _list.writeToFile(); - } - - void reportFailure() { _failed = true; } - }; -} - -bool -MountPointList::verifyHealthyDisks(int mountPointCount) -{ - WriteStatusFileIfFailing statusWriter(*this); - int usable = 0, empty = 0; - std::map lackingChunkDef; - // Test disks and get chunkinfo - for (uint32_t i=0, n=_mountPoints.size(); igetDirectory(ost.str(), i); - dir->addEvent(Device::NOT_FOUND, - "Disk not found during scanning of disks directory", - VESPA_STRLOC); - _mountPoints[i] = dir; - statusWriter.reportFailure(); - } - if (dir->isOk()) { - testMountPoint(*dir); - if (!dir->isOk()) statusWriter.reportFailure(); - } - // Don't touch unhealthy or non-existing disks. - if (!dir->isOk()) { - std::ostringstream ost; - ost << "Not using disk " << i << " marked bad: "; - dir->getLastEvent()->print(ost, true, " "); - LOG(warning, "%s", ost.str().c_str()); - continue; - } - - // Read chunkinfo - using vespalib::IoException; - Chunk chunk; - try{ - chunk = getChunkDef(dir->getPath()); - } catch (IoException& e) { - chunk = Chunk(); - if (e.getType() == IoException::NOT_FOUND) { - if (!emptyDir(*dir)) { - dir->addEvent(Device::INTERNAL_FAILURE, - "Foreign data in mountpoint. New " - "mountpoints added should be empty.", ""); - } - } else { - LOG(warning, "Failed to read chunkinfo file from mountpoint %s", - dir->getPath().c_str()); - Device::State newState(getDeviceState(e.getType())); - if (newState != Device::OK) { - dir->addEvent(newState, e.what(), VESPA_STRLOC); - } - } - } catch (std::exception& e) { - LOG(warning, "Failed to read chunkinfo file from mountpoint %s", - dir->getPath().c_str()); - dir->addEvent(Device::INTERNAL_FAILURE, e.what(), VESPA_STRLOC); - } - - // If disk was found unusable, don't use it. - if (!dir->isOk()) { - LOG(warning, "Unusable disk %d: %s", - i, dir->getLastEvent()->toString(true).c_str()); - statusWriter.reportFailure(); - continue; - } - ++usable; - // Ensure disk fits in with the already detected ones. - if (!chunk.valid()) { - ++empty; - lackingChunkDef[i] = dir; - } else if (chunk.nr != i) { - std::ostringstream ost; - ost << "Disk " << dir->getPath() << " thinks it's disk " << chunk.nr - << " (instead of " << i << ")."; - LOG(error, "%s", ost.str().c_str()); - throw vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); - } else if (mountPointCount == -1) { - mountPointCount = chunk.total; - } else if (static_cast(mountPointCount) != chunk.total) { - std::ostringstream ost; - ost << "Disk " << dir->getPath() << " thinks it's disk " << chunk.nr - << " of " << chunk.total << " (instead of " << i << " of " - << mountPointCount << ")."; - LOG(error, "%s", ost.str().c_str()); - throw vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); - } - } - if (empty == usable && usable != mountPointCount && mountPointCount != -1) { - std::ostringstream ost; - ost << "Found " << usable << " disks and config says we're " - << "supposed to have " << mountPointCount << ". Not initializing " - << "disks."; - throw vespalib::IllegalStateException(ost.str(), VESPA_STRLOC); - } - bool retval = true; - // Handle case where no chunkinfo file present (none/unusable/new disks) - if (mountPointCount == -1) { - if (_mountPoints.size() == 0) { - LOG(error, "No disks configured for storage node. Disk " - "directories/symlinks for this node should be created " - "in %s/disks/. Please refer to VDS documentation to " - "learn how to add disks", _vdsRoot.c_str()); - throw spi::HandledException("No disks configured", VESPA_STRLOC); - } else if (usable == 0) { - LOG(error, "All of the configured disks are unusable. " - "Please refer to previous warnings and the VDS " - "documentation for troubleshooting"); - throw spi::HandledException("All disks unusable", VESPA_STRLOC); - } else { - mountPointCount = _mountPoints.size(); - LOG(info, "All disks empty. Setting up node to run with the %u " - "found disks.", mountPointCount); - retval = false; - } - } - // Write chunkdef files where these are missing - for (std::map::const_iterator it - = lackingChunkDef.begin(); it != lackingChunkDef.end(); ++it) - { - const Directory::SP &dir = it->second; - Chunk c; - c.nr = it->first; - c.total = mountPointCount; - if (c.nr >= c.total) { - LOG(warning, "Can't use disk %u of %u as the index is too high. " - "(Disks are indexed from zero)", c.nr, c.total); - continue; - } - if (!emptyDir(*dir)) { - LOG(warning, "Not creating chunkinfo file on disk %u as it already " - "contains data. If you want to include the disk, " - "create chunkinfo file manually.", c.nr); - assert(!dir->isOk()); - continue; - } - using vespalib::IoException; - try{ - writeChunkDef(c, dir->getPath()); - retval = true; - } catch (IoException& e) { - statusWriter.reportFailure(); - LOG(warning, "Failed to write chunkinfo file to mountpoint %s.", - dir->getPath().c_str()); - Device::State newState(getDeviceState(e.getType())); - if (newState != Device::OK) { - dir->addEvent(newState, e.what(), VESPA_STRLOC); - } - } catch (std::exception& e) { - statusWriter.reportFailure(); - LOG(warning, "Failed to write chunkinfo file to mountpoint %s", - dir->getPath().c_str()); - dir->addEvent(Device::INTERNAL_FAILURE, e.what(), VESPA_STRLOC); - } - } - // If we need more entries in mountpointlist, due to chunkinfo - // showing more indexes, add them. - for (int i = _mountPoints.size(); i < mountPointCount; ++i) { - std::ostringstream ost; - ost << _vdsRoot + "/disks/d" << i; - Directory::SP dir(_deviceManager->getDirectory(ost.str(), i)); - dir->addEvent(Device::NOT_FOUND, - "Disk not found during scanning of disks directory", - VESPA_STRLOC); - _mountPoints.push_back(dir); - } - if (static_cast(_mountPoints.size()) > mountPointCount) { - _mountPoints.resize(mountPointCount); - } - return retval; -} - -uint16_t -MountPointList::findIndex(const Directory& dir) const -{ - for (uint16_t i = 0; i < _mountPoints.size(); ++i) { - if (_mountPoints[i].get() != 0 && dir == *_mountPoints[i]) return i; - } - throw vespalib::IllegalArgumentException( - "Could not find directory " + dir.toString(), VESPA_STRLOC); -} - -std::string -MountPointList::getDiskStatusFileName() const -{ - return _vdsRoot + "/disks.status"; -} - -vespalib::string -MountPointList::reportXmlStatus(vespalib::xml::XmlOutputStream& xos, - const framework::HttpUrlPath&) const -{ - xos << *_deviceManager; - return ""; -} - -} diff --git a/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.h b/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.h deleted file mode 100644 index 350e8f6e1d5..00000000000 --- a/memfilepersistence/src/vespa/memfilepersistence/device/mountpointlist.h +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * \class storage::MountPointList - * \ingroup persistence - * - * \brief Class holding information about the mount points used by storage - * - * We need to keep a list of mount points, to read and write the mount point - * file, and to access what mount points should be used and not. - * - * NOTE: A mountpoint is often referred to as a disk, even though you technicly - * can have multiple mountpoints per partition and multiple partitions per disk. - * - * IMPORTANT: Remember to call verifyHealthyDisks() before starting to use them. - */ - -#pragma once - -#include "devicemanager.h" -#include "directory.h" -#include -#include -#include - -namespace storage { -namespace lib { - class NodeState; -} - -namespace memfile { - -struct MountPointList : public framework::XmlStatusReporter { - typedef std::unique_ptr UP; - - /** Create a mount point list. */ - MountPointList(const std::string& vdsRoot, - const std::vector& diskPath, - std::unique_ptr); - ~MountPointList(); - - DeviceManager& getDeviceManager() { return *_deviceManager; } - - /** - * Call init to initialize the mount point list in the regular fashion. - * @param diskCount Number of disks to find, or 0 to auto-detect. - * @return The number of usable disks found. - */ - void init(uint16_t diskCount); - - /** - * Initialize the disks, see description of diskPath config in - * stor-devices. Will be called as part of init(). - */ - void initDisks(); - - /** - * Scan disks directory for disks. Add entries found, which does not exist, - * or are marked NOT_FOUND to this instance. - * - * To prevent reading from possible bad disks, we cannot access the disks - * themselves. Thus, in case of symlinks, it assumes the symlink is to a - * directory. - */ - void scanForDisks(); - - /** - * Read the disk status file and adjust the list. - * Important that any entry marking a disk bad (except for NOT_FOUND if it - * should be in the file) overrides any disks marked ok in this instance. - * - * Similarily to scanForDisks(), this does not access the disks itself. - */ - void readFromFile(); - - /** - * Initialize the partition monitors within the partitions. Done after - * partition creation, as partition objects are generated for bad disks. - */ - void initializePartitionMonitors(); - - /** - * Write the current state of disks to the disk status file. - * Disks that are OK or NOT_FOUND does not need to be written to file. - */ - void writeToFile() const; - - /** - * Go through all the mountpoints marked ok, and check that they work. - *