diff options
author | Harald Musum <musum@yahoo-inc.com> | 2017-06-14 09:47:35 +0200 |
---|---|---|
committer | Harald Musum <musum@yahoo-inc.com> | 2017-06-14 09:47:35 +0200 |
commit | e2d88dc91681376e12550c869a77ef64fab2327e (patch) | |
tree | 0c152b99efd32778715a8c76eb2b292162cb1219 | |
parent | 09d16433d3b83133a3bdea0ec8785c0e60ed9082 (diff) | |
parent | 3073e29384e0b67ba7e191f0406c24869c9e0153 (diff) |
Merge branch 'master' into hmusum/16-distribution-bits-only-for-prod
68 files changed, 450 insertions, 2176 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java index e983f794e76..051f0e9e52e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/InstanceResolver.java @@ -59,9 +59,8 @@ class InstanceResolver { ConfigDefinitionKey defKey = new ConfigDefinitionKey(key); try { if (targetDef != null) applyDef(builder, targetDef); - ConfigInstance instance = getInstance(defKey, builder.getClass().getClassLoader()); - Class<? extends ConfigInstance> clazz = instance.getClass(); - return clazz.getConstructor(new Class<?>[]{builder.getClass()}).newInstance(builder); + Class<? extends ConfigInstance> clazz = getConfigClass(defKey, builder.getClass().getClassLoader()); + return clazz.getConstructor(builder.getClass()).newInstance(builder); } catch (Exception e) { throw new ConfigurationRuntimeException(e); } @@ -158,27 +157,20 @@ class InstanceResolver { * @param cKey a ConfigKey * @return a {@link ConfigInstance} or null if not available in classpath */ - private static ConfigInstance getInstance(ConfigDefinitionKey cKey, ClassLoader instanceLoader) { + @SuppressWarnings("unchecked") + private static Class<? extends ConfigInstance> getConfigClass(ConfigDefinitionKey cKey, ClassLoader instanceLoader) { String className = ConfigGenerator.createClassName(cKey.getName()); - Class<?> clazz; String fullClassName = packageName(cKey) + "." + className; + Class<?> clazz; try { clazz = instanceLoader != null ? instanceLoader.loadClass(fullClassName) : Class.forName(fullClassName); } catch (ClassNotFoundException e) { - return null; - } - Object i; - try { - Constructor<?> configConstructor = clazz.getDeclaredConstructor(); - configConstructor.setAccessible(true); - i = configConstructor.newInstance(); - } catch (InvocationTargetException | InstantiationException | IllegalAccessException | NoSuchMethodException e) { - throw new ConfigurationRuntimeException(e); + throw new ConfigurationRuntimeException("Could not find config class for key " + cKey, e); } - if (!(i instanceof ConfigInstance)) { - throw new ConfigurationRuntimeException(fullClassName + " is not a ConfigInstance, can not produce config for the name '" + cKey.getName() + "'."); + if (! ConfigInstance.class.isAssignableFrom(clazz)) { + throw new ConfigurationRuntimeException(fullClassName + " is not a ConfigInstance subclass, can not produce config for " + cKey); } - return (ConfigInstance) i; + return (Class<? extends ConfigInstance>) clazz; } static String packageName(ConfigDefinitionKey cKey) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index ff8bf61cdb6..8227be3347a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -102,7 +102,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri ModelElement documentsElement = contentElement.getChild("documents"); Map<String, NewDocumentType> documentDefinitions = - new SearchDefinitionBuilder().build(context.getParentProducer().getRoot().getDeployState().getDocumentModel().getDocumentManager(), documentsElement); + new SearchDefinitionBuilder().build(context.getDeployState().getDocumentModel().getDocumentManager(), documentsElement); String routingSelection = new DocumentSelectionBuilder().build(documentsElement); Redundancy redundancy = new RedundancyBuilder().build(contentElement); @@ -110,7 +110,7 @@ public class ContentCluster extends AbstractConfigProducer implements StorDistri ContentCluster c = new ContentCluster(context.getParentProducer(), getClusterName(contentElement), documentDefinitions, globallyDistributedDocuments, routingSelection, redundancy, - context.getParentProducer().getRoot().getDeployState().getProperties().zone()); + context.getDeployState().getProperties().zone()); c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterName(contentElement), contentElement).build(c, contentElement.getXml()); c.search = new ContentSearchCluster.Builder(documentDefinitions, globallyDistributedDocuments).build(c, contentElement.getXml()); c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index 5e02a6a84d1..38c7e8524b4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -210,6 +210,7 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P public static class Compression { public enum Type { NONE("NONE"), + ZSTD("ZSTD"), LZ4("LZ4"); public final String name; diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index 66e97e76aa4..88527951ccc 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -361,6 +361,6 @@ TuningIoOptionsLight = string "normal" | string "directio" TuningIoOptionsFull = string "normal" | string "directio" | string "mmap" | string "mlock" TuningCompression = element compression { - element type { string "none" | string "lz4" }? & + element type { string "none" | string "lz4" | string "zstd" }? & element level { xsd:nonNegativeInteger }? } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java index 595013c506e..1ac892f551a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/ContentClusterUtils.java @@ -2,10 +2,8 @@ package com.yahoo.vespa.model.content.utils; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.api.HostProvisioner; -import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; @@ -53,7 +51,7 @@ public class ContentClusterUtils { public static ContentCluster createCluster(String clusterXml, MockRoot root) throws Exception { Document doc = XML.getDocument(clusterXml); Admin admin = new Admin(root, new Yamas("vespa", 60), new Metrics(), Collections.emptyMap(), false); - ConfigModelContext context = ConfigModelContext.create(null, DeployState.createTestState(), null, root, null); + ConfigModelContext context = ConfigModelContext.create(null, root.getDeployState(), null, root, null); return new ContentCluster.Builder(admin).build(Collections.emptyList(), context, doc.getDocumentElement()); } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provisioning/package-info.java b/config-provisioning/src/main/java/com/yahoo/config/provisioning/package-info.java new file mode 100644 index 00000000000..ff01235ac6f --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provisioning/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.config.provisioning; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-provisioning/src/main/resources/configdefinitions/node-repository.def b/config-provisioning/src/main/resources/configdefinitions/node-repository.def index 872f2e50f9c..5fa1838af66 100644 --- a/config-provisioning/src/main/resources/configdefinitions/node-repository.def +++ b/config-provisioning/src/main/resources/configdefinitions/node-repository.def @@ -2,4 +2,4 @@ namespace=config.provisioning # Docker image to use in REST API responses. This must be a fully qualified name, including registry, but excluding # version. Example: my-docker-registry.domain.tld:8080/dist/vespa -dockerImage string +dockerImage string default="dummyImage" diff --git a/configserver/src/main/sh/start-filedistribution b/configserver/src/main/sh/start-filedistribution index df8dce34f13..ecf04e9e035 100755 --- a/configserver/src/main/sh/start-filedistribution +++ b/configserver/src/main/sh/start-filedistribution @@ -78,5 +78,5 @@ if [ "$multitenant" = "true" ]; then export VESPA_LOG_CONTROL_DIR export VESPA_LOG_CONTROL_FILE cd ${ROOT} - vespa-runserver -r 30 -s filedistributor -p $PIDFILE_FILEDISTRIBUTOR -- ${ROOT}/sbin/vespa-filedistributor --configid $VESPA_CONFIG_ID + vespa-runserver -r 30 -s filedistributor -p $PIDFILE_FILEDISTRIBUTOR -- ${ROOT}/sbin/filedistributor --configid $VESPA_CONFIG_ID fi diff --git a/configserver/src/main/sh/start-logd b/configserver/src/main/sh/start-logd index 3ad735d5183..ab5f0384678 100644 --- a/configserver/src/main/sh/start-logd +++ b/configserver/src/main/sh/start-logd @@ -68,5 +68,5 @@ if [ "$multitenant" = "true" ]; then PIDFILE_LOGD=var/run/logd.pid VESPA_SERVICE_NAME=logd export VESPA_SERVICE_NAME - vespa-runserver -r 30 -s logd -p $PIDFILE_LOGD -- ${ROOT}/sbin/logd + vespa-runserver -r 30 -s logd -p $PIDFILE_LOGD -- ${ROOT}/sbin/vespa-logd fi diff --git a/dist/post_install.sh b/dist/post_install.sh index 897f19a592e..95f07b76f1c 100755 --- a/dist/post_install.sh +++ b/dist/post_install.sh @@ -69,10 +69,6 @@ ln -s $PREFIX/lib/jars/zkfacade-jar-with-dependencies.jar $INSTALLPATH/conf/conf ln -s $PREFIX/conf/configserver-app/components $INSTALLPATH/lib/jars/config-models ln -s vespa-storaged-bin $INSTALLPATH/sbin/vespa-distributord-bin -# Temporary when renaming programs in filedistribution -ln -s vespa-filedistributor $INSTALLPATH/sbin/filedistributor -ln -s vespa-filedistributor-bin $INSTALLPATH/sbin/filedistributor-bin - # Temporary when renaming binaries in fnet ln -s vespa-rpc-info $INSTALLPATH/bin/rpc_info ln -s vespa-rpc-invoke $INSTALLPATH/bin/rpc_invoke @@ -117,6 +113,9 @@ ln -s vespa-logctl $INSTALLPATH/bin/logctl ln -s vespa-logfmt $INSTALLPATH/bin/logfmt ln -s vespa-log-replay $INSTALLPATH/bin/vlogreplay +# Temporary when renaming binaries in logd +ln -s vespa-logd $INSTALLPATH/sbin/logd + # Temporary when renaming binaries in memfilepersistence ln -s vespa-dump-slotfile $INSTALLPATH/bin/dumpslotfile ln -s vespa-vds-disktool-bin $INSTALLPATH/bin/vdsdisktool-bin @@ -148,3 +147,6 @@ ln -s vespa-slobrok-cmd $INSTALLPATH/bin/sbcmd # Temporary when renaming programs in configd ln -s vespa-run-as-vespa-user $INSTALLPATH/bin/run-as-yahoo ln -s vespa-config-sentinel $INSTALLPATH/sbin/config-sentinel + +# Temporary when renaming programs in logserver +ln -s vespa-logserver-start $INSTALLPATH/bin/logserver-start diff --git a/dist/vespa.spec b/dist/vespa.spec index b97662a3df5..1b4bc307468 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -22,6 +22,7 @@ BuildRequires: devtoolset-6-binutils BuildRequires: Judy-devel BuildRequires: cmake3 BuildRequires: lz4-devel +BuildRequires: libzstd-devel BuildRequires: zlib-devel BuildRequires: maven BuildRequires: libicu-devel @@ -30,23 +31,24 @@ BuildRequires: java-1.8.0-openjdk-devel BuildRequires: openssl-devel BuildRequires: rpm-build BuildRequires: make -BuildRequires: vespa-boost-devel >= 1.59 -BuildRequires: vespa-cppunit-devel >= 1.12.1 -BuildRequires: vespa-libtorrent-devel >= 1.0.9 -BuildRequires: vespa-zookeeper-c-client-devel >= 3.4.8 +BuildRequires: vespa-boost-devel >= 1.59.0-6 +BuildRequires: vespa-cppunit-devel >= 1.12.1-6 +BuildRequires: vespa-libtorrent-devel >= 1.0.11-6 +BuildRequires: vespa-zookeeper-c-client-devel >= 3.4.9-6 BuildRequires: systemd Requires: epel-release Requires: Judy Requires: lz4 +Requires: libzstd Requires: zlib Requires: libicu Requires: llvm3.9 Requires: java-1.8.0-openjdk Requires: openssl -Requires: vespa-boost >= 1.59 -Requires: vespa-cppunit >= 1.12.1 -Requires: vespa-libtorrent >= 1.0.9 -Requires: vespa-zookeeper-c-client >= 3.4.8 +Requires: vespa-boost >= 1.59.0-6 +Requires: vespa-cppunit >= 1.12.1-6 +Requires: vespa-libtorrent >= 1.0.11-6 +Requires: vespa-zookeeper-c-client >= 3.4.9-6 Requires(pre): shadow-utils # Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private diff --git a/docker/Dockerfile.build b/docker/Dockerfile.build index a0b935093eb..4e1d5d8d4cb 100644 --- a/docker/Dockerfile.build +++ b/docker/Dockerfile.build @@ -8,11 +8,13 @@ RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vesp devtoolset-6-gcc-c++ \ devtoolset-6-libatomic-devel \ devtoolset-6-binutils \ + git \ make \ cmake3 \ ccache \ Judy-devel \ lz4-devel \ + libzstd-devel \ zlib-devel \ maven \ libicu-devel \ diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci new file mode 100644 index 00000000000..39a5317fe14 --- /dev/null +++ b/docker/Dockerfile.ci @@ -0,0 +1,11 @@ +FROM vespabuild + +COPY vespa-ci-internal.sh /vespa-ci-internal.sh + +# Build using non-root user 'vespabuilder' +RUN useradd --create-home -s /bin/bash vespabuilder +WORKDIR /home/vespabuilder +USER vespabuilder + +# Java requires proper locale for unicode +ENV LANG en_US.UTF-8 diff --git a/docker/Dockerfile.run b/docker/Dockerfile.run index 5f0dd1fd0cc..9de5953194e 100644 --- a/docker/Dockerfile.run +++ b/docker/Dockerfile.run @@ -8,6 +8,7 @@ RUN yum-config-manager --add-repo https://copr.fedorainfracloud.org/coprs/g/vesp Judy \ lz4 \ zlib \ + libzstd \ libicu \ llvm3.9 \ java-1.8.0-openjdk \ diff --git a/docker/README.md b/docker/README.md index 393d726cff2..e3105b9263a 100644 --- a/docker/README.md +++ b/docker/README.md @@ -13,6 +13,10 @@ Execute ```./build-vespa.sh <Vespa version number>``` to build Vespa from this s The produced rpms will be available in this folder after compiliation. The version number will be compiled into binaries, but has no other meaning than that. +## Building and testing Vespa +Execute ```./vespa-ci.sh <git commit>``` to build and test a specific branch/tag/commit. + + ## Running Vespa Execute ```./run-vespa.sh <Vespa version number>``` to start Vespa. diff --git a/docker/vespa-ci-internal.sh b/docker/vespa-ci-internal.sh new file mode 100755 index 00000000000..3d7cc439528 --- /dev/null +++ b/docker/vespa-ci-internal.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 <git commit>" + echo "This script should not be called manually." + exit 1 +fi + +GIT_COMMIT=$1 +SOURCE_DIR=/home/vespabuilder/vespa +BUILD_DIR=/home/vespabuilder/build +THREADS=$(nproc --all) + +mkdir "${SOURCE_DIR}" +mkdir "${BUILD_DIR}" +git clone --no-checkout --local --no-hardlinks file:///vespa "${SOURCE_DIR}" +cd "${SOURCE_DIR}" +git checkout ${GIT_COMMIT} +source /opt/rh/devtoolset-6/enable || true +sh ./bootstrap.sh full +mvn install +cd "${BUILD_DIR}" +cmake3 -DCMAKE_INSTALL_PREFIX=/opt/vespa \ + -DJAVA_HOME=/usr/lib/jvm/java-openjdk \ + -DEXTRA_LINK_DIRECTORY="/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib64/llvm3.9/lib" \ + -DEXTRA_INCLUDE_DIRECTORY="/opt/vespa-boost/include;/opt/vespa-libtorrent/include;/opt/vespa-zookeeper-c-client/include;/opt/vespa-cppunit/include;/usr/include/llvm3.9" \ + -DCMAKE_INSTALL_RPATH="/opt/vespa/lib64;/opt/vespa-boost/lib;/opt/vespa-libtorrent/lib;/opt/vespa-zookeeper-c-client/lib;/opt/vespa-cppunit/lib;/usr/lib/jvm/java-1.8.0/jre/lib/amd64/server;/usr/include/llvm3.9" \ + -DCMAKE_BUILD_RPATH=/opt/vespa/lib64 \ + "${SOURCE_DIR}" +make -j ${THREADS} +make -j ${THREADS} test diff --git a/docker/vespa-ci.sh b/docker/vespa-ci.sh new file mode 100755 index 00000000000..2d17634a409 --- /dev/null +++ b/docker/vespa-ci.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 <git commit>" + exit 1 +fi + +DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) +cd $DIR + +GIT_COMMIT=$1 +BUILD_DOCKER_IMAGE="vespabuild" +CI_DOCKER_IMAGE="vespaci" + +docker build -t "$BUILD_DOCKER_IMAGE" -f Dockerfile.build . +docker build -t "$CI_DOCKER_IMAGE" -f Dockerfile.ci . +docker run --rm -v $(pwd)/..:/vespa --entrypoint /vespa-ci-internal.sh "$CI_DOCKER_IMAGE" "$GIT_COMMIT" diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index c0089c2bed8..cc4784c1ee2 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -10,6 +10,7 @@ vespa_define_module( EXTERNAL_DEPENDS lz4 + zstd LIBS src/vespa/document diff --git a/document/src/tests/serialization/compression_test.cpp b/document/src/tests/serialization/compression_test.cpp index b41dfad7b36..240a309d180 100644 --- a/document/src/tests/serialization/compression_test.cpp +++ b/document/src/tests/serialization/compression_test.cpp @@ -1,27 +1,44 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Unit tests for annotation serialization. -#include <vespa/log/log.h> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/document/util/compressor.h> +#include <vespa/vespalib/data/databuffer.h> +#include <vespa/log/log.h> LOG_SETUP("compression_test"); using namespace document; +using namespace document::compression; using namespace vespalib; static vespalib::string _G_compressableText("AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" - "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE"); + "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" + "XYZABCDEFGHIJGJMNOPQRSTUVW" + "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" + "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" + "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" + "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" + "AAAAAAAAAAAAAAABBBBBBBBBBBBBBBBBCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCDDDDDDDDDDDDDDDDDDDDDDDDDEEEEEEEEEEEEEEE" + "XYZABCDEFGHIJGJMNOPQRSTUVW"); TEST("requireThatLZ4CompressFine") { CompressionConfig cfg(CompressionConfig::Type::LZ4); ConstBufferRef ref(_G_compressableText.c_str(), _G_compressableText.size()); DataBuffer compressed; EXPECT_EQUAL(CompressionConfig::Type::LZ4, compress(cfg, ref, compressed, false)); + EXPECT_EQUAL(66u, compressed.getDataLen()); +} + +TEST("requireThatZStdCompressFine") { + CompressionConfig cfg(CompressionConfig::Type::ZSTD); + ConstBufferRef ref(_G_compressableText.c_str(), _G_compressableText.size()); + DataBuffer compressed; + EXPECT_EQUAL(CompressionConfig::Type::ZSTD, compress(cfg, ref, compressed, false)); + EXPECT_EQUAL(64u, compressed.getDataLen()); } TEST_MAIN() { diff --git a/document/src/vespa/document/fieldvalue/serializablearray.cpp b/document/src/vespa/document/fieldvalue/serializablearray.cpp index c8beea5e79c..0fb85a2aaf2 100644 --- a/document/src/vespa/document/fieldvalue/serializablearray.cpp +++ b/document/src/vespa/document/fieldvalue/serializablearray.cpp @@ -2,14 +2,15 @@ #include "serializablearray.h" #include <vespa/document/util/serializableexceptions.h> #include <vespa/document/util/bytebuffer.h> +#include <vespa/document/util/compressor.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/data/databuffer.h> -#include <vespa/document/util/compressor.h> #include <vespa/log/log.h> LOG_SETUP(".document.serializable-array"); using std::vector; +using vespalib::make_string; namespace document { @@ -179,6 +180,7 @@ SerializableArray::clear(int id) void SerializableArray::deCompress() // throw (DeserializeException) { + using document::compression::decompress; // will only do this once LOG_ASSERT(_compSerData); @@ -201,15 +203,14 @@ SerializableArray::deCompress() // throw (DeserializeException) false); } catch (const std::runtime_error & e) { throw DeserializeException( - vespalib::make_string( "Document was compressed with code unknown code %d", _serializedCompression), + make_string( "Document was compressed with code unknown code %d", _serializedCompression), VESPA_STRLOC); } if (unCompressed.getDataLen() != (size_t)_uncompressedLength) { throw DeserializeException( - vespalib::make_string( - "Did not decompress to the expected length: had %" PRIu64 ", wanted %d, got %" PRIu64, - _compSerData->getRemaining(), _uncompressedLength, unCompressed.getDataLen()), + make_string("Did not decompress to the expected length: had %zu, wanted %d, got %zu", + _compSerData->getRemaining(), _uncompressedLength, unCompressed.getDataLen()), VESPA_STRLOC); } assert(newSerialization->getBuffer() == unCompressed.getData()); diff --git a/document/src/vespa/document/serialization/vespadocumentserializer.cpp b/document/src/vespa/document/serialization/vespadocumentserializer.cpp index 1f6a4a4f0b0..a95dd092f20 100644 --- a/document/src/vespa/document/serialization/vespadocumentserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentserializer.cpp @@ -263,6 +263,7 @@ vespalib::ConstBufferRef compressStream(const CompressionConfig &config, nbostream &stream, vespalib::DataBuffer & compressed_data) { + using compression::compress; vespalib::ConstBufferRef buf(stream.c_str(), stream.size()); if (config.useCompression() && bigEnough(stream.size(), config)) { CompressionConfig::Type compressedType = compress(config, vespalib::ConstBufferRef(stream.c_str(), stream.size()), compressed_data, false); diff --git a/document/src/vespa/document/util/CMakeLists.txt b/document/src/vespa/document/util/CMakeLists.txt index d8fe0d833ff..ae1f7441062 100644 --- a/document/src/vespa/document/util/CMakeLists.txt +++ b/document/src/vespa/document/util/CMakeLists.txt @@ -1,11 +1,13 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(document_util OBJECT SOURCES - stringutil.cpp bytebuffer.cpp - serializable.cpp compressor.cpp + lz4compressor.cpp + zstdcompressor.cpp printable.cpp + serializable.cpp + stringutil.cpp DEPENDS AFTER document_documentconfig diff --git a/document/src/vespa/document/util/compressionconfig.h b/document/src/vespa/document/util/compressionconfig.h index 1ed6ec882ec..ff02e35977a 100644 --- a/document/src/vespa/document/util/compressionconfig.h +++ b/document/src/vespa/document/util/compressionconfig.h @@ -7,16 +7,16 @@ namespace document { - struct CompressionConfig { enum Type { - NONE = 0, - HISTORIC_1 = 1, - HISTORIC_2 = 2, - HISTORIC_3 = 3, - HISTORIC_4 = 4, - UNCOMPRESSABLE = 5, - LZ4 = 6 + NONE = 0, + HISTORIC_1 = 1, + HISTORIC_2 = 2, + HISTORIC_3 = 3, + HISTORIC_4 = 4, + UNCOMPRESSABLE = 5, + LZ4 = 6, + ZSTD = 7 }; CompressionConfig() @@ -47,6 +47,7 @@ struct CompressionConfig { case 4: return HISTORIC_4; case 5: return UNCOMPRESSABLE; case 6: return LZ4; + case 7: return ZSTD; default: return NONE; } } diff --git a/document/src/vespa/document/util/compressor.cpp b/document/src/vespa/document/util/compressor.cpp index 0783bb28a60..6ada9bca3db 100644 --- a/document/src/vespa/document/util/compressor.cpp +++ b/document/src/vespa/document/util/compressor.cpp @@ -1,56 +1,17 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "compressor.h" + +#include "lz4compressor.h" +#include "zstdcompressor.h" #include <vespa/vespalib/util/memory.h> #include <vespa/vespalib/util/stringfmt.h> -#include <stdexcept> -#include <lz4.h> -#include <lz4hc.h> +#include <vespa/vespalib/data/databuffer.h> using vespalib::alloc::Alloc; using vespalib::ConstBufferRef; using vespalib::DataBuffer; using vespalib::make_string; -namespace document -{ - -size_t LZ4Compressor::adjustProcessLen(uint16_t, size_t len) const { return LZ4_compressBound(len); } -size_t LZ4Compressor::adjustUnProcessLen(uint16_t, size_t len) const { return len; } - -bool -LZ4Compressor::process(const CompressionConfig& config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) -{ - const char * input(static_cast<const char *>(inputV)); - char * output(static_cast<char *>(outputV)); - int sz(-1); - int maxOutputLen = LZ4_compressBound(inputLen); - if (config.compressionLevel > 6) { - Alloc state = Alloc::alloc(LZ4_sizeofStateHC()); - sz = LZ4_compress_HC_extStateHC(state.get(), input, output, inputLen, maxOutputLen, config.compressionLevel); - } else { - Alloc state = Alloc::alloc(LZ4_sizeofState()); - sz = LZ4_compress_fast_extState(state.get(), input, output, inputLen, maxOutputLen, 1); - } - if (sz != 0) { - outputLenV = sz; - } - assert(sz != 0); - return (sz != 0); - -} - -bool -LZ4Compressor::unprocess(const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) -{ - const char * input(static_cast<const char *>(inputV)); - char * output(static_cast<char *>(outputV)); - int sz = LZ4_decompress_safe(input, output, inputLen, outputLenV); - if (sz > 0) { - outputLenV = sz; - } - assert(sz > 0); - return (sz > 0); -} +namespace document::compression { CompressionConfig::Type compress(ICompressor & compressor, const CompressionConfig & compression, const ConstBufferRef & org, DataBuffer & dest) @@ -78,6 +39,12 @@ docompress(const CompressionConfig & compression, const ConstBufferRef & org, Da type = compress(lz4, compression, org, dest); } break; + case CompressionConfig::ZSTD: + { + ZStdCompressor zstd; + type = compress(zstd, compression, org, dest); + } + break; case CompressionConfig::NONE: default: break; @@ -138,6 +105,12 @@ decompress(const CompressionConfig::Type & type, size_t uncompressedLen, const C decompress(lz4, uncompressedLen, org, dest, allowSwap); } break; + case CompressionConfig::ZSTD: + { + ZStdCompressor zstd; + decompress(zstd, uncompressedLen, org, dest, allowSwap); + } + break; case CompressionConfig::NONE: case CompressionConfig::UNCOMPRESSABLE: if (allowSwap) { @@ -154,4 +127,15 @@ decompress(const CompressionConfig::Type & type, size_t uncompressedLen, const C } } +size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t payloadSize) { + if (type == CompressionConfig::LZ4) { + document::LZ4Compressor lz4; + return lz4.adjustProcessLen(0, payloadSize); + } else if (type == CompressionConfig::ZSTD) { + document::ZStdCompressor zstd; + return zstd.adjustProcessLen(0, payloadSize); + } + return payloadSize; +} + } diff --git a/document/src/vespa/document/util/compressor.h b/document/src/vespa/document/util/compressor.h index 599d9424889..11103beb93c 100644 --- a/document/src/vespa/document/util/compressor.h +++ b/document/src/vespa/document/util/compressor.h @@ -1,12 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/document/util/compressionconfig.h> -#include <vespa/vespalib/data/databuffer.h> +#include "compressionconfig.h" #include <vespa/vespalib/util/buffer.h> -namespace document -{ +namespace vespalib { class DataBuffer; } + +namespace document { class ICompressor { @@ -15,17 +15,9 @@ public: virtual bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) = 0; virtual bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) = 0; virtual size_t adjustProcessLen(uint16_t options, size_t len) const = 0; - virtual size_t adjustUnProcessLen(uint16_t options, size_t len) const = 0; }; -class LZ4Compressor : public ICompressor -{ -public: - bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) override; - bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) override; - size_t adjustProcessLen(uint16_t options, size_t len) const override; - size_t adjustUnProcessLen(uint16_t options, size_t len) const override; -}; +namespace compression { /** * Will try to compress a buffer according to the config. If the criteria can not @@ -51,5 +43,9 @@ CompressionConfig::Type compress(const CompressionConfig & compression, const ve */ void decompress(const CompressionConfig::Type & compression, size_t uncompressedLen, const vespalib::ConstBufferRef & org, vespalib::DataBuffer & dest, bool allowSwap); + +size_t computeMaxCompressedsize(CompressionConfig::Type type, size_t uncompressedSize); + } +} diff --git a/document/src/vespa/document/util/lz4compressor.cpp b/document/src/vespa/document/util/lz4compressor.cpp new file mode 100644 index 00000000000..9b41df239a9 --- /dev/null +++ b/document/src/vespa/document/util/lz4compressor.cpp @@ -0,0 +1,46 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "lz4compressor.h" +#include <vespa/vespalib/util/alloc.h> +#include <lz4.h> +#include <lz4hc.h> +#include <cassert> + +using vespalib::alloc::Alloc; + +namespace document { + +size_t LZ4Compressor::adjustProcessLen(uint16_t, size_t len) const { return LZ4_compressBound(len); } + +bool +LZ4Compressor::process(const CompressionConfig& config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) +{ + const char * input(static_cast<const char *>(inputV)); + char * output(static_cast<char *>(outputV)); + int sz(-1); + int maxOutputLen = LZ4_compressBound(inputLen); + if (config.compressionLevel > 6) { + Alloc state = Alloc::alloc(LZ4_sizeofStateHC()); + sz = LZ4_compress_HC_extStateHC(state.get(), input, output, inputLen, maxOutputLen, config.compressionLevel); + } else { + Alloc state = Alloc::alloc(LZ4_sizeofState()); + sz = LZ4_compress_fast_extState(state.get(), input, output, inputLen, maxOutputLen, 1); + } + assert(sz != 0); + outputLenV = sz; + return (sz != 0); + +} + +bool +LZ4Compressor::unprocess(const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) +{ + const char * input(static_cast<const char *>(inputV)); + char * output(static_cast<char *>(outputV)); + int sz = LZ4_decompress_safe(input, output, inputLen, outputLenV); + assert(sz > 0); + outputLenV = sz; + return (sz > 0); +} + +} diff --git a/document/src/vespa/document/util/lz4compressor.h b/document/src/vespa/document/util/lz4compressor.h new file mode 100644 index 00000000000..aa1726cd639 --- /dev/null +++ b/document/src/vespa/document/util/lz4compressor.h @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "compressor.h" + +namespace document { + +class LZ4Compressor : public ICompressor +{ +public: + bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) override; + bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) override; + size_t adjustProcessLen(uint16_t options, size_t len) const override; +}; + +} + diff --git a/document/src/vespa/document/util/zstdcompressor.cpp b/document/src/vespa/document/util/zstdcompressor.cpp new file mode 100644 index 00000000000..bb13d8009c1 --- /dev/null +++ b/document/src/vespa/document/util/zstdcompressor.cpp @@ -0,0 +1,34 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "zstdcompressor.h" +#include <vespa/vespalib/util/alloc.h> +#include <zstd.h> +#include <cassert> + +using vespalib::alloc::Alloc; + +namespace document { + +size_t ZStdCompressor::adjustProcessLen(uint16_t, size_t len) const { return ZSTD_compressBound(len); } + +bool +ZStdCompressor::process(const CompressionConfig& config, const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) +{ + size_t maxOutputLen = ZSTD_compressBound(inputLen); + size_t sz = ZSTD_compress(outputV, maxOutputLen, inputV, inputLen, config.compressionLevel); + assert( ! ZSTD_isError(sz) ); + outputLenV = sz; + return ! ZSTD_isError(sz); + +} + +bool +ZStdCompressor::unprocess(const void * inputV, size_t inputLen, void * outputV, size_t & outputLenV) +{ + size_t sz = ZSTD_decompress(outputV, outputLenV, inputV, inputLen); + assert( ! ZSTD_isError(sz) ); + outputLenV = sz; + return ! ZSTD_isError(sz); +} + +} diff --git a/document/src/vespa/document/util/zstdcompressor.h b/document/src/vespa/document/util/zstdcompressor.h new file mode 100644 index 00000000000..5c52c144651 --- /dev/null +++ b/document/src/vespa/document/util/zstdcompressor.h @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "compressor.h" + +namespace document { + +class ZStdCompressor : public ICompressor +{ +public: + bool process(const CompressionConfig& config, const void * input, size_t inputLen, void * output, size_t & outputLen) override; + bool unprocess(const void * input, size_t inputLen, void * output, size_t & outputLen) override; + size_t adjustProcessLen(uint16_t options, size_t len) const override; +}; + +} + diff --git a/filedistribution/src/apps/filedistributor/.gitignore b/filedistribution/src/apps/filedistributor/.gitignore index 89aaa70637e..a41b1963b70 100644 --- a/filedistribution/src/apps/filedistributor/.gitignore +++ b/filedistribution/src/apps/filedistributor/.gitignore @@ -1,2 +1,2 @@ /filedistributor -vespa-filedistributor-bin +filedistributor-bin diff --git a/filedistribution/src/apps/filedistributor/CMakeLists.txt b/filedistribution/src/apps/filedistributor/CMakeLists.txt index 42e47856208..1a7ef87bbac 100644 --- a/filedistribution/src/apps/filedistributor/CMakeLists.txt +++ b/filedistribution/src/apps/filedistributor/CMakeLists.txt @@ -2,7 +2,7 @@ vespa_add_executable(filedistribution_filedistributor_app SOURCES filedistributor.cpp - OUTPUT_NAME vespa-filedistributor-bin + OUTPUT_NAME filedistributor-bin INSTALL sbin DEPENDS filedistribution_distributor diff --git a/filedistribution/src/apps/status/.gitignore b/filedistribution/src/apps/status/.gitignore index 2105a3c7051..6dc1c1fff5d 100644 --- a/filedistribution/src/apps/status/.gitignore +++ b/filedistribution/src/apps/status/.gitignore @@ -1,2 +1,2 @@ -/vespa-status-filedistribution-bin +/status-filedistribution filedistribution_status-filedistribution_app diff --git a/filedistribution/src/apps/status/CMakeLists.txt b/filedistribution/src/apps/status/CMakeLists.txt index e46e0c58d86..63e9752496e 100644 --- a/filedistribution/src/apps/status/CMakeLists.txt +++ b/filedistribution/src/apps/status/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(filedistribution_status-filedistribution_app SOURCES status-filedistribution.cpp INSTALL bin - OUTPUT_NAME vespa-status-filedistribution-bin + OUTPUT_NAME status-filedistribution DEPENDS filedistribution_filedistributionmodel filedistribution_common diff --git a/filedistribution/src/apps/status/vespa-status-filedistribution.sh b/filedistribution/src/apps/status/vespa-status-filedistribution.sh index 1ba56d6c798..104688a4883 100644 --- a/filedistribution/src/apps/status/vespa-status-filedistribution.sh +++ b/filedistribution/src/apps/status/vespa-status-filedistribution.sh @@ -65,4 +65,4 @@ ROOT=${VESPA_HOME%/} ZKSTRING=$($ROOT/libexec/vespa/vespa-config.pl -zkstring) test -z "$VESPA_LOG_LEVEL" && VESPA_LOG_LEVEL=warning export VESPA_LOG_LEVEL -exec $ROOT/bin/vespa-status-filedistribution-bin --zkstring "$ZKSTRING" $@ +exec $ROOT/bin/status-filedistribution --zkstring "$ZKSTRING" $@ diff --git a/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java b/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java index 64cdd9da416..7965cf34ccf 100644 --- a/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java +++ b/filedistributionmanager/src/main/java/com/yahoo/vespa/filedistribution/FileDistributionManager.java @@ -1,10 +1,14 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.filedistribution; +import com.yahoo.system.execution.ProcessExecutor; +import com.yahoo.system.execution.ProcessResult; + import java.io.File; import java.io.IOException; import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.concurrent.locks.Lock; /** @@ -109,10 +113,19 @@ public class FileDistributionManager { } public void reloadDeployFileDistributor() { + String binaryName = "filedistributor"; + String command = "pkill -SIGUSR1 -x " + binaryName; + int timeoutSeconds = 10; try (LockGuard guard = new LockGuard(lock)) { - Runtime.getRuntime().exec("pkill -SIGUSR1 -x filedistributor"); + Optional<ProcessResult> result = new ProcessExecutor.Builder(timeoutSeconds).build().execute(command); + if (! result.isPresent()) { + throw new RuntimeException(String.format("Executing '%s' timed out", command)); + } else if (result.get().exitCode != 0) { + throw new RuntimeException(String.format("Error when executing '%s', exit code %d: %s\n%s", + command, result.get().exitCode, result.get().stdOut, result.get().stdErr)); + } } catch (IOException e) { - throw new RuntimeException("Failed to reinitialize the filedistributor", e); + throw new RuntimeException("Failed to reinitialize " + binaryName, e); } } diff --git a/install_java.cmake b/install_java.cmake index 00de96a69ad..c4a7ab3d99e 100644 --- a/install_java.cmake +++ b/install_java.cmake @@ -50,7 +50,7 @@ install_fat_java_artifact(vespaclient-java) install_fat_java_artifact(zkfacade) vespa_install_script(application-preprocessor/src/main/sh/vespa-preprocess-application bin) -vespa_install_script(jdisc_core/src/main/perl/jdisc_logfmt bin) +vespa_install_script(jdisc_core/src/main/perl/vespa-jdisc-logfmt bin) vespa_install_script(config-proxy/src/main/sh/vespa-config-ctl bin) vespa_install_script(config-proxy/src/main/sh/vespa-config-loadtester bin) vespa_install_script(config-proxy/src/main/sh/vespa-config-verification bin) @@ -70,12 +70,12 @@ vespa_install_script(vespaclient-java/src/main/sh/vespa-get.sh vespa-get bin) vespa_install_script(vespaclient-java/src/main/sh/vespa-visit.sh vespa-visit bin) vespa_install_script(vespaclient-java/src/main/sh/vespa-visit-target.sh vespa-visit-target bin) -vespa_install_script(logserver/bin/logserver-start.sh logserver-start bin) +vespa_install_script(logserver/bin/logserver-start.sh vespa-logserver-start bin) install(DIRECTORY config-model/src/main/resources/schema DESTINATION share/vespa PATTERN ".gitignore" EXCLUDE) install(DIRECTORY config-model/src/main/resources/schema DESTINATION share/vespa/schema/version/6.x PATTERN ".gitignore" EXCLUDE) -install(FILES jdisc_core/src/main/perl/jdisc_logfmt.1 DESTINATION man/man1) +install(FILES jdisc_core/src/main/perl/vespa-jdisc-logfmt.1 DESTINATION man/man1) install(FILES config-model-fat/src/main/resources/config-models.xml diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index b289d5816cd..3206e70ad34 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -295,9 +295,9 @@ <goal>exec</goal> </goals> <configuration> - <executable>${project.basedir}/src/test/perl/jdisc_logfmt_test.sh</executable> + <executable>${project.basedir}/src/test/perl/vespa-jdisc-logfmt-test.sh</executable> <arguments> - <argument>${project.basedir}/src/main/perl/jdisc_logfmt</argument> + <argument>${project.basedir}/src/main/perl/vespa-jdisc-logfmt</argument> </arguments> </configuration> </execution> diff --git a/jdisc_core/src/main/perl/jdisc_logfmt b/jdisc_core/src/main/perl/vespa-jdisc-logfmt index 11a21cbe01a..11a21cbe01a 100755 --- a/jdisc_core/src/main/perl/jdisc_logfmt +++ b/jdisc_core/src/main/perl/vespa-jdisc-logfmt diff --git a/jdisc_core/src/main/perl/jdisc_logfmt.1 b/jdisc_core/src/main/perl/vespa-jdisc-logfmt.1 index 0a884c0ec85..c79ee2165f2 100644 --- a/jdisc_core/src/main/perl/jdisc_logfmt.1 +++ b/jdisc_core/src/main/perl/vespa-jdisc-logfmt.1 @@ -1,11 +1,11 @@ .\" $Id: logfmt.1,v 1.12 2007-06-19 09:37:25 daljord Exp $ .\" .Dd October 29, 2004 -.Dt LOGFMT \&1 "JDisc documentation" +.Dt VESPA-JDISC-LOGFMT \&1 "JDisc documentation" .Os "Yahoo! JDisc" "2.3" .Os .Sh NAME -.Nm logfmt +.Nm vespa-jdisc-logfmt .Nd select and format messages from JDisc log files .Sh SYNOPSIS .Nm @@ -73,7 +73,7 @@ is shown. Print the time in human-readable [YYYY-MM-DD HH:mm:ss] format. Note that the time is printed in the local timezone; to get GMT output use -.Nm "\*[q]env TZ=GMT logfmt\*[q]" +.Nm "\*[q]env TZ=GMT vespa-jdisc-logfmt\*[q]" as your command. .It msecs Add milliseconds after the seconds in @@ -151,7 +151,7 @@ truncating if necessary. The command: .Pp .Bd -literal -offset indent -logfmt -l event -s service,message,fmttime,message +vespa-jdisc-logfmt -l event -s service,message,fmttime,message .Ed .Pp will display only messages with log level "event", @@ -175,7 +175,7 @@ order is. The command: .Pp .Bd -literal -offset indent -logfmt -l all-info,-debug -s level \e +vespa-jdisc-logfmt -l all-info,-debug -s level \e -s time,usecs,component,message -t -l -event .Ed .Pp diff --git a/jdisc_core/src/test/perl/help.Levent.expected b/jdisc_core/src/test/perl/help.Levent.expected index b35e2d9c36b..1f7c17cd81c 100644 --- a/jdisc_core/src/test/perl/help.Levent.expected +++ b/jdisc_core/src/test/perl/help.Levent.expected @@ -1,4 +1,4 @@ -Usage: jdisc_logfmt [options] [inputfile ...] +Usage: vespa-jdisc-logfmt [options] [inputfile ...] Options: -l LEVELLIST --level=LEVELLIST select levels to include -L LEVELLIST --add-level=LEVELLIST define extra levels diff --git a/jdisc_core/src/test/perl/help.expected b/jdisc_core/src/test/perl/help.expected index 58da8183696..cadddb4cf63 100644 --- a/jdisc_core/src/test/perl/help.expected +++ b/jdisc_core/src/test/perl/help.expected @@ -1,4 +1,4 @@ -Usage: jdisc_logfmt [options] [inputfile ...] +Usage: vespa-jdisc-logfmt [options] [inputfile ...] Options: -l LEVELLIST --level=LEVELLIST select levels to include -L LEVELLIST --add-level=LEVELLIST define extra levels diff --git a/jdisc_core/src/test/perl/jdisc_logfmt_test.sh b/jdisc_core/src/test/perl/vespa-jdisc-logfmt-test.sh index 0166d33d8ab..106db534ee3 100755 --- a/jdisc_core/src/test/perl/jdisc_logfmt_test.sh +++ b/jdisc_core/src/test/perl/vespa-jdisc-logfmt-test.sh @@ -5,9 +5,9 @@ DIFF=/usr/bin/diff LOGFMT=${1} if which perl &> /dev/null; then - echo "Running jdisc_logfmt test suite." + echo "Running vespa-jdisc-logfmt test suite." else - echo "Ignoring jdisc_logfmt test suite as there is no perl executable." + echo "Ignoring vespa-jdisc-logfmt test suite as there is no perl executable." exit 0 fi diff --git a/jdisc_http_service/docs/class-diagram.graffle b/jdisc_http_service/docs/class-diagram.graffle deleted file mode 100644 index 938459c6571..00000000000 --- a/jdisc_http_service/docs/class-diagram.graffle +++ /dev/null @@ -1,1856 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> -<plist version="1.0"> -<dict> - <key>ActiveLayerIndex</key> - <integer>0</integer> - <key>ApplicationVersion</key> - <array> - <string>com.omnigroup.OmniGrafflePro</string> - <string>139.7.0.167456</string> - </array> - <key>AutoAdjust</key> - <true/> - <key>BackgroundGraphic</key> - <dict> - <key>Bounds</key> - <string>{{0, 0}, {558.99999713897705, 783}}</string> - <key>Class</key> - <string>SolidGraphic</string> - <key>ID</key> - <integer>2</integer> - <key>Style</key> - <dict> - <key>shadow</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - <key>stroke</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - </dict> - </dict> - <key>BaseZoom</key> - <integer>0</integer> - <key>CanvasOrigin</key> - <string>{0, 0}</string> - <key>ColumnAlign</key> - <integer>1</integer> - <key>ColumnSpacing</key> - <real>36</real> - <key>CreationDate</key> - <string>2012-06-18 12:41:37 +0000</string> - <key>Creator</key> - <string>Einar Rosenvinge</string> - <key>DisplayScale</key> - <string>1.000 cm = 1.000 cm</string> - <key>GraphDocumentVersion</key> - <integer>8</integer> - <key>GraphicsList</key> - <array> - <dict> - <key>Bounds</key> - <string>{{404.66666889190674, 39.999999999999936}, {151.17318725585938, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>ID</key> - <integer>114</integer> - <key>Magnets</key> - <array> - <string>{0, 1}</string> - <string>{0, -1}</string> - <string>{1, 0}</string> - <string>{-1, 0}</string> - </array> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>Color</key> - <dict> - <key>b</key> - <string>0</string> - <key>g</key> - <string>1</string> - <key>r</key> - <string>0</string> - </dict> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 J2SE API}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>TableGroup</string> - <key>Graphics</key> - <array> - <dict> - <key>Bounds</key> - <string>{{404.66666889190674, 18.66666666666665}, {151.17318725585938, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>109</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - <key>stroke</key> - <dict> - <key>Color</key> - <dict> - <key>b</key> - <string>0</string> - <key>g</key> - <string>0</string> - <key>r</key> - <string>1</string> - </dict> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 Netty API}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - </array> - <key>ID</key> - <integer>108</integer> - </dict> - <dict> - <key>Bounds</key> - <string>{{404.66666666666731, 7.9472862957175039e-08}, {151.17318725585938, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>ID</key> - <integer>107</integer> - <key>Magnets</key> - <array> - <string>{0, 1}</string> - <string>{0, -1}</string> - <string>{1, 0}</string> - <string>{-1, 0}</string> - </array> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>Color</key> - <dict> - <key>b</key> - <string>1</string> - <key>g</key> - <string>0</string> - <key>r</key> - <string>0</string> - </dict> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 jDISC core API}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - </dict> - <dict> - <key>Bounds</key> - <string>{{328.46341597965198, 296.25884156306495}, {29.333332061767578, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>106</integer> - <key>Line</key> - <dict> - <key>ID</key> - <integer>105</integer> - <key>Offset</key> - <real>6.6666665077209473</real> - <key>Position</key> - <real>0.91559326648712158</real> - <key>RotationType</key> - <integer>0</integer> - </dict> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - <key>shadow</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - <key>stroke</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Align</key> - <integer>0</integer> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 1}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>LineGraphic</string> - <key>Head</key> - <dict> - <key>ID</key> - <integer>52</integer> - </dict> - <key>ID</key> - <integer>105</integer> - <key>Points</key> - <array> - <string>{225.55682373046864, 284.810302734375}</string> - <string>{352.43905966196957, 312.07829430296113}</string> - </array> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>HeadArrow</key> - <string>StickArrow</string> - <key>Legacy</key> - <true/> - <key>LineType</key> - <integer>1</integer> - <key>TailArrow</key> - <string>0</string> - </dict> - </dict> - <key>Tail</key> - <dict> - <key>ID</key> - <integer>104</integer> - </dict> - </dict> - <dict> - <key>Bounds</key> - <string>{{101.99999999999997, 277.810302734375}, {123.55682373046864, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>ID</key> - <integer>104</integer> - <key>Magnets</key> - <array> - <string>{0, 1}</string> - <string>{0, -1}</string> - <string>{1, 0}</string> - <string>{-1, 0}</string> - </array> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>Color</key> - <dict> - <key>b</key> - <string>0</string> - <key>g</key> - <string>0</string> - <key>r</key> - <string>1</string> - </dict> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 ChannelPipeline}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - </dict> - <dict> - <key>Bounds</key> - <string>{{166.25436339285309, 474.88842165638926}, {29.333332061767578, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>103</integer> - <key>Line</key> - <dict> - <key>ID</key> - <integer>102</integer> - <key>Offset</key> - <real>6.6666665077209473</real> - <key>Position</key> - <real>0.78203368186950684</real> - <key>RotationType</key> - <integer>0</integer> - </dict> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - <key>shadow</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - <key>stroke</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Align</key> - <integer>0</integer> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 1}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>LineGraphic</string> - <key>Head</key> - <dict> - <key>ID</key> - <integer>89</integer> - </dict> - <key>ID</key> - <integer>102</integer> - <key>Points</key> - <array> - <string>{291.52424638132226, 513.75253787937902}</string> - <string>{148.21246360738095, 481.32189037141075}</string> - </array> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>HeadArrow</key> - <string>StickArrow</string> - <key>Legacy</key> - <true/> - <key>LineType</key> - <integer>1</integer> - <key>TailArrow</key> - <string>0</string> - </dict> - </dict> - <key>Tail</key> - <dict> - <key>ID</key> - <integer>58</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>LineGraphic</string> - <key>Head</key> - <dict> - <key>ID</key> - <integer>95</integer> - </dict> - <key>ID</key> - <integer>96</integer> - <key>Points</key> - <array> - <string>{116.77841269969511, 466.71157835576878}</string> - <string>{116.77840998702608, 420.16666668003933}</string> - </array> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>HeadArrow</key> - <string>Arrow</string> - <key>Legacy</key> - <true/> - <key>LineType</key> - <integer>1</integer> - <key>TailArrow</key> - <string>0</string> - </dict> - </dict> - <key>Tail</key> - <dict> - <key>ID</key> - <integer>89</integer> - </dict> - </dict> - <dict> - <key>Bounds</key> - <string>{{58.945077896118171, 405.66666666666708}, {115.66666412353516, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>ID</key> - <integer>95</integer> - <key>Magnets</key> - <array> - <string>{0, 1}</string> - <string>{0, -1}</string> - <string>{1, 0}</string> - <string>{-1, 0}</string> - </array> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>Color</key> - <dict> - <key>b</key> - <string>1</string> - <key>g</key> - <string>0</string> - <key>r</key> - <string>0</string> - </dict> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 Request}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>TableGroup</string> - <key>Graphics</key> - <array> - <dict> - <key>Bounds</key> - <string>{{68.3333333333334, 467.21157836914102}, {96.890159606933594, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>89</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 HttpRequest}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - </array> - <key>ID</key> - <integer>88</integer> - </dict> - <dict> - <key>Class</key> - <string>LineGraphic</string> - <key>Head</key> - <dict> - <key>ID</key> - <integer>86</integer> - </dict> - <key>ID</key> - <integer>87</integer> - <key>Points</key> - <array> - <string>{350.25724339020428, 513.73666184812191}</string> - <string>{432.54926154576208, 492.26520134230651}</string> - </array> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>HeadArrow</key> - <string>Arrow</string> - <key>Legacy</key> - <true/> - <key>Pattern</key> - <integer>1</integer> - <key>TailArrow</key> - <string>0</string> - </dict> - </dict> - <key>Tail</key> - <dict> - <key>ID</key> - <integer>58</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>TableGroup</string> - <key>Graphics</key> - <array> - <dict> - <key>Bounds</key> - <string>{{429.24670918782579, 464.13918876647955}, {115.08661651611328, 28}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>86</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - <key>stroke</key> - <dict> - <key>Color</key> - <dict> - <key>b</key> - <string>1</string> - <key>g</key> - <string>0</string> - <key>r</key> - <string>0</string> - </dict> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 <<interface>>\ -ResponseHandler}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - </array> - <key>ID</key> - <integer>85</integer> - </dict> - <dict> - <key>Class</key> - <string>LineGraphic</string> - <key>Head</key> - <dict> - <key>ID</key> - <integer>80</integer> - </dict> - <key>ID</key> - <integer>81</integer> - <key>Points</key> - <array> - <string>{388.98464357649044, 311.73041212162735}</string> - <string>{421.12154403577119, 242.99627753494852}</string> - </array> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>HeadArrow</key> - <string>Arrow</string> - <key>Legacy</key> - <true/> - <key>LineType</key> - <integer>1</integer> - <key>TailArrow</key> - <string>0</string> - </dict> - </dict> - <key>Tail</key> - <dict> - <key>ID</key> - <integer>52</integer> - </dict> - </dict> - <dict> - <key>Bounds</key> - <string>{{298.33331664403278, 228.54334004720087}, {246, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>ID</key> - <integer>80</integer> - <key>Magnets</key> - <array> - <string>{0, 1}</string> - <string>{0, -1}</string> - <string>{1, 0}</string> - <string>{-1, 0}</string> - </array> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>Color</key> - <dict> - <key>b</key> - <string>0</string> - <key>g</key> - <string>0</string> - <key>r</key> - <string>1</string> - </dict> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 IdleStateAwareChannelUpstreamHandler}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>LineGraphic</string> - <key>Head</key> - <dict> - <key>ID</key> - <integer>10</integer> - </dict> - <key>ID</key> - <integer>27</integer> - <key>Points</key> - <array> - <string>{129.27841644847507, 130.04334003445351}</string> - <string>{129.27841313680011, 80.666667938232422}</string> - </array> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>HeadArrow</key> - <string>Arrow</string> - <key>Legacy</key> - <true/> - <key>LineType</key> - <integer>1</integer> - <key>TailArrow</key> - <string>0</string> - </dict> - </dict> - <key>Tail</key> - <dict> - <key>ID</key> - <integer>69</integer> - </dict> - </dict> - <dict> - <key>Bounds</key> - <string>{{71.445081075032547, 66.666667938232422}, {115.66666412353516, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>ID</key> - <integer>10</integer> - <key>Magnets</key> - <array> - <string>{0, 1}</string> - <string>{0, -1}</string> - <string>{1, 0}</string> - <string>{-1, 0}</string> - </array> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>Color</key> - <dict> - <key>b</key> - <string>1</string> - <key>g</key> - <string>0</string> - <key>r</key> - <string>0</string> - </dict> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 AbstractResource}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>LineGraphic</string> - <key>Head</key> - <dict> - <key>ID</key> - <integer>78</integer> - </dict> - <key>ID</key> - <integer>79</integer> - <key>Points</key> - <array> - <string>{185.26276724928567, 130.48075966792658}</string> - <string>{297.8372493866778, 116.27917789513803}</string> - </array> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>HeadArrow</key> - <string>Arrow</string> - <key>Legacy</key> - <true/> - <key>Pattern</key> - <integer>1</integer> - <key>TailArrow</key> - <string>0</string> - </dict> - </dict> - <key>Tail</key> - <dict> - <key>ID</key> - <integer>69</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>TableGroup</string> - <key>Graphics</key> - <array> - <dict> - <key>Bounds</key> - <string>{{298.3333059188999, 92.666666666666913}, {151.17318725585938, 28}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>78</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - <key>stroke</key> - <dict> - <key>Color</key> - <dict> - <key>b</key> - <string>0</string> - <key>g</key> - <string>0</string> - <key>r</key> - <string>1</string> - </dict> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 <<interface>>\ -ChannelPipelineFactory}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - </array> - <key>ID</key> - <integer>77</integer> - </dict> - <dict> - <key>Class</key> - <string>LineGraphic</string> - <key>Head</key> - <dict> - <key>ID</key> - <integer>75</integer> - </dict> - <key>ID</key> - <integer>76</integer> - <key>Points</key> - <array> - <string>{402.97819417613408, 311.99312587103537}</string> - <string>{465.1473485992222, 286.41782928501391}</string> - </array> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>HeadArrow</key> - <string>Arrow</string> - <key>Legacy</key> - <true/> - <key>Pattern</key> - <integer>1</integer> - <key>TailArrow</key> - <string>0</string> - </dict> - </dict> - <key>Tail</key> - <dict> - <key>ID</key> - <integer>52</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>TableGroup</string> - <key>Graphics</key> - <array> - <dict> - <key>Bounds</key> - <string>{{454.91339111328136, 258.22751967112259}, {89.419929504394531, 28}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>75</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - <key>stroke</key> - <dict> - <key>Color</key> - <dict> - <key>b</key> - <string>0</string> - <key>g</key> - <string>1</string> - <key>r</key> - <string>0</string> - </dict> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 <<interface>>\ -Runnable}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - </array> - <key>ID</key> - <integer>74</integer> - </dict> - <dict> - <key>Class</key> - <string>LineGraphic</string> - <key>Head</key> - <dict> - <key>ID</key> - <integer>34</integer> - </dict> - <key>ID</key> - <integer>9</integer> - <key>Points</key> - <array> - <string>{143.15706944536032, 130.31230832708621}</string> - <string>{250.79491585172687, 74.231022194055214}</string> - </array> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>HeadArrow</key> - <string>Arrow</string> - <key>Legacy</key> - <true/> - <key>Pattern</key> - <integer>1</integer> - <key>TailArrow</key> - <string>0</string> - </dict> - </dict> - <key>Tail</key> - <dict> - <key>ID</key> - <integer>69</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>TableGroup</string> - <key>Graphics</key> - <array> - <dict> - <key>Bounds</key> - <string>{{226.66667683919275, 45.999999999999929}, {102.88706970214844, 28}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>34</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - <key>stroke</key> - <dict> - <key>Color</key> - <dict> - <key>b</key> - <string>1</string> - <key>g</key> - <string>0</string> - <key>r</key> - <string>0</string> - </dict> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 <<interface>>\ -ServerProvider}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - </array> - <key>ID</key> - <integer>33</integer> - </dict> - <dict> - <key>Bounds</key> - <string>{{344.6666666666668, 438.85001627604163}, {17, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>YES</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>63</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - <key>shadow</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - <key>stroke</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Align</key> - <integer>0</integer> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 1}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>Wrap</key> - <string>NO</string> - </dict> - <dict> - <key>Bounds</key> - <string>{{308.27841186523432, 496.00000000000023}, {29.333332061767578, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>62</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - <key>shadow</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - <key>stroke</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Align</key> - <integer>0</integer> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 1..*}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>LineGraphic</string> - <key>Head</key> - <dict> - <key>ID</key> - <integer>58</integer> - </dict> - <key>ID</key> - <integer>61</integer> - <key>Points</key> - <array> - <string>{364.20702685721608, 438.63027009978003}</string> - <string>{326.6915789284576, 513.41623846111793}</string> - </array> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>HeadArrow</key> - <string>0</string> - <key>Legacy</key> - <true/> - <key>LineType</key> - <integer>1</integer> - <key>TailArrow</key> - <string>0</string> - </dict> - </dict> - <key>Tail</key> - <dict> - <key>ID</key> - <integer>54</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>TableGroup</string> - <key>Graphics</key> - <array> - <dict> - <key>Bounds</key> - <string>{{226.66666603088379, 513.86289469401004}, {192.55682373046875, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>58</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 RequestContext}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - <dict> - <key>Bounds</key> - <string>{{226.66666603088379, 527.86289469401004}, {192.55682373046875, 42}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>59</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Align</key> - <integer>0</integer> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 HttpRequest request\ -ContentChannel requestContent\ -ContentChannel responseContent}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - <dict> - <key>Bounds</key> - <string>{{226.66666603088379, 569.86289469401004}, {192.55682373046875, 28}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>60</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Align</key> - <integer>0</integer> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 connect()\ -handleResponse(Response)}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - </array> - <key>GridH</key> - <array> - <integer>58</integer> - <integer>59</integer> - <integer>60</integer> - <array/> - </array> - <key>ID</key> - <integer>57</integer> - </dict> - <dict> - <key>Class</key> - <string>TableGroup</string> - <key>Graphics</key> - <array> - <dict> - <key>Bounds</key> - <string>{{226.66666666666663, 312.183349609375}, {317.66665649414062, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>52</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 ChannelContext}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - <dict> - <key>Bounds</key> - <string>{{226.66666666666652, 326.183349609375}, {317.66665649414062, 28}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>53</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Align</key> - <integer>0</integer> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 LinkedBlockingQueue<ResponsePart> responseOutputs\ -Channel serverChannel}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - <dict> - <key>Bounds</key> - <string>{{226.66666666666663, 354.183349609375}, {317.66665649414062, 84}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>54</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Align</key> - <integer>0</integer> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 channelConnected()\ -messageReceived()\ -exceptionCaught()\ -channelDisconnected()\ -channelIdle()\ -run()}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - </array> - <key>GridH</key> - <array> - <integer>52</integer> - <integer>53</integer> - <integer>54</integer> - <array/> - </array> - <key>ID</key> - <integer>51</integer> - </dict> - <dict> - <key>Bounds</key> - <string>{{158.97158196265923, 262.43389980796718}, {29.333332061767578, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>47</integer> - <key>Line</key> - <dict> - <key>ID</key> - <integer>16</integer> - <key>Offset</key> - <real>12.666667938232422</real> - <key>Position</key> - <real>0.93361091613769531</real> - <key>RotationType</key> - <integer>0</integer> - </dict> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - <key>shadow</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - <key>stroke</key> - <dict> - <key>Draws</key> - <string>NO</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Align</key> - <integer>0</integer> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 0..*}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>LineGraphic</string> - <key>Head</key> - <dict> - <key>ID</key> - <integer>104</integer> - </dict> - <key>ID</key> - <integer>16</integer> - <key>Points</key> - <array> - <string>{144.7864237041758, 242.98231480726633}</string> - <string>{163.77841186523429, 277.810302734375}</string> - </array> - <key>Style</key> - <dict> - <key>stroke</key> - <dict> - <key>HeadArrow</key> - <string>StickArrow</string> - <key>Legacy</key> - <true/> - <key>LineType</key> - <integer>1</integer> - <key>TailArrow</key> - <string>0</string> - </dict> - </dict> - <key>Tail</key> - <dict> - <key>ID</key> - <integer>71</integer> - </dict> - </dict> - <dict> - <key>Class</key> - <string>TableGroup</string> - <key>Graphics</key> - <array> - <dict> - <key>Bounds</key> - <string>{{33.000005086262888, 130.54334004720062}, {192.55682373046875, 14}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>69</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720\qc - -\f0\b\fs24 \cf0 HttpServer}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - <dict> - <key>Bounds</key> - <string>{{33.000005086262888, 144.54334004720062}, {192.55682373046875, 42}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>70</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Align</key> - <integer>0</integer> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 Channel serverChannel\ -Executor channelWorkerExecutor\ -HttpServerConfig config}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - <dict> - <key>Bounds</key> - <string>{{33.000005086262888, 186.54334004720062}, {192.55682373046875, 56}}</string> - <key>Class</key> - <string>ShapedGraphic</string> - <key>FitText</key> - <string>Vertical</string> - <key>Flow</key> - <string>Resize</string> - <key>ID</key> - <integer>71</integer> - <key>Shape</key> - <string>Rectangle</string> - <key>Style</key> - <dict> - <key>fill</key> - <dict> - <key>GradientCenter</key> - <string>{-0.29411799999999999, -0.264706}</string> - </dict> - </dict> - <key>Text</key> - <dict> - <key>Align</key> - <integer>0</integer> - <key>Text</key> - <string>{\rtf1\ansi\ansicpg1252\cocoartf1138\cocoasubrtf470 -{\fonttbl\f0\fswiss\fcharset0 Helvetica;} -{\colortbl;\red255\green255\blue255;} -\pard\tx560\tx1120\tx1680\tx2240\tx2800\tx3360\tx3920\tx4480\tx5040\tx5600\tx6160\tx6720 - -\f0\fs24 \cf0 start()\ -close()\ -destroy()\ -getPipeline()}</string> - <key>VerticalPad</key> - <integer>0</integer> - </dict> - <key>TextPlacement</key> - <integer>0</integer> - </dict> - </array> - <key>GridH</key> - <array> - <integer>69</integer> - <integer>70</integer> - <integer>71</integer> - <array/> - </array> - <key>ID</key> - <integer>68</integer> - </dict> - </array> - <key>GridInfo</key> - <dict/> - <key>GuidesLocked</key> - <string>NO</string> - <key>GuidesVisible</key> - <string>YES</string> - <key>HPages</key> - <integer>1</integer> - <key>ImageCounter</key> - <integer>1</integer> - <key>KeepToScale</key> - <false/> - <key>Layers</key> - <array> - <dict> - <key>Lock</key> - <string>NO</string> - <key>Name</key> - <string>Layer 1</string> - <key>Print</key> - <string>YES</string> - <key>View</key> - <string>YES</string> - </dict> - </array> - <key>LayoutInfo</key> - <dict> - <key>Animate</key> - <string>NO</string> - <key>circoMinDist</key> - <real>18</real> - <key>circoSeparation</key> - <real>0.0</real> - <key>layoutEngine</key> - <string>dot</string> - <key>neatoSeparation</key> - <real>0.0</real> - <key>twopiSeparation</key> - <real>0.0</real> - </dict> - <key>LinksVisible</key> - <string>NO</string> - <key>MagnetsVisible</key> - <string>NO</string> - <key>MasterSheets</key> - <array/> - <key>ModificationDate</key> - <string>2012-06-19 09:20:43 +0000</string> - <key>Modifier</key> - <string>Einar Rosenvinge</string> - <key>NotesVisible</key> - <string>NO</string> - <key>Orientation</key> - <integer>2</integer> - <key>OriginVisible</key> - <string>NO</string> - <key>PageBreaks</key> - <string>YES</string> - <key>PrintInfo</key> - <dict> - <key>NSBottomMargin</key> - <array> - <string>float</string> - <string>41</string> - </array> - <key>NSHorizonalPagination</key> - <array> - <string>coded</string> - <string>BAtzdHJlYW10eXBlZIHoA4QBQISEhAhOU051bWJlcgCEhAdOU1ZhbHVlAISECE5TT2JqZWN0AIWEASqEhAFxlwCG</string> - </array> - <key>NSLeftMargin</key> - <array> - <string>float</string> - <string>18</string> - </array> - <key>NSPaperSize</key> - <array> - <string>size</string> - <string>{594.99999713897705, 842}</string> - </array> - <key>NSPrintReverseOrientation</key> - <array> - <string>int</string> - <string>0</string> - </array> - <key>NSRightMargin</key> - <array> - <string>float</string> - <string>18</string> - </array> - <key>NSTopMargin</key> - <array> - <string>float</string> - <string>18</string> - </array> - </dict> - <key>PrintOnePage</key> - <false/> - <key>ReadOnly</key> - <string>NO</string> - <key>RowAlign</key> - <integer>1</integer> - <key>RowSpacing</key> - <real>36</real> - <key>SheetTitle</key> - <string>Canvas 1</string> - <key>SmartAlignmentGuidesActive</key> - <string>YES</string> - <key>SmartDistanceGuidesActive</key> - <string>YES</string> - <key>UniqueID</key> - <integer>1</integer> - <key>UseEntirePage</key> - <false/> - <key>VPages</key> - <integer>1</integer> - <key>WindowInfo</key> - <dict> - <key>CurrentSheet</key> - <integer>0</integer> - <key>ExpandedCanvases</key> - <array> - <dict> - <key>name</key> - <string>Canvas 1</string> - </dict> - </array> - <key>Frame</key> - <string>{{246, 375}, {1064, 803}}</string> - <key>ListView</key> - <true/> - <key>OutlineWidth</key> - <integer>142</integer> - <key>RightSidebar</key> - <false/> - <key>ShowRuler</key> - <true/> - <key>Sidebar</key> - <true/> - <key>SidebarWidth</key> - <integer>120</integer> - <key>VisibleRegion</key> - <string>{{-25, 0}, {610, 442.66666666666669}}</string> - <key>Zoom</key> - <real>1.5</real> - <key>ZoomValues</key> - <array> - <array> - <string>Canvas 1</string> - <real>1.5</real> - <real>0.25</real> - </array> - </array> - </dict> -</dict> -</plist> diff --git a/jdisc_http_service/docs/class-diagram.png b/jdisc_http_service/docs/class-diagram.png Binary files differdeleted file mode 100644 index ebccfd75bf9..00000000000 --- a/jdisc_http_service/docs/class-diagram.png +++ /dev/null diff --git a/jdisc_http_service/docs/httpserver.html b/jdisc_http_service/docs/httpserver.html deleted file mode 100644 index 66afeb687fe..00000000000 --- a/jdisc_http_service/docs/httpserver.html +++ /dev/null @@ -1,96 +0,0 @@ -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" -<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> - "http://www.w3.org/TR/html4/loose.dtd"> -<html> -<head> - <title>HTTP Server Architecture</title> - <style type="text/css"> - body { - font: 13px/1.231 arial,helvetica,clean,sans-serif; - *font-size: small; - *font: x-small; - } - select,input,button,textarea { - font: 99% arial,helvetica,clean,sans-serif; - } - table{ - font-size: inherit; - font: 100%; - } - pre,code,kbd,samp,tt { - font-family: monospace; - *font-size: 108%; - line-height: 100%; - } - </style> -</head> -<body> -<p>The HTTP server is started by calling <code>HttpServer.start()</code>, which in turn calls <code>ServerBootstrap.bind()</code> - provided by - Netty.</p> - -<img src="class-diagram.png" alt="Class diagram"> - -<p>Since our HttpServer implements <code>ChannelPipelineFactory</code> (provided by jetty), its - <code>getPipeline()</code> method is called for every new channel that is connected. There is hence a one-to-many - relationship between a <code>HttpServer</code> and a pipeline (and a one-to-one relationship between an actual - channel and a pipeline).</p> - -<p>The pipeline is responsible for decoding (and possibly deflating etc.) every new request that is received on a - channel. The final element in the pipeline is a <code>ChannelContext</code>, which is the jDISC class for handling - requests on a channel.</p> - -<p>The <code>ChannelContext</code> implements <code>SimpleChannelUpstreamHandler</code> (provided by Jetty), which has - simple callback methods for various event types. <br/>Examples:</p> - -<ul> - <li><code>channelConnected()</code></li> - <li><code>channelDisconnected()</code></li> - <li><code>messageReceived()</code></li> -</ul> - -<p>Since <code>ChannelContext</code> supports HTTP keep-alive and HTTP pipelining, it needs to keep track of multiple - requests made on the channel, and their order.</p> - -<p>In <code>messageReceived()</code> it will:</p> -<ul> - <li>Determine if the element received is a new HTTP request, or a chunk belonging to the previous one.</li> - <li>If it's a request, create a DISC <code>Request</code> object for it, and call <code>Request.connect()</code>, - which will in turn give it to the actual application, through the use of - <code>RequestHandler.handleRequest()</code>. - </li> - <li>If it's a chunk, fetch the previously added <code>RequestContext</code>, and use it to write the data received - into the <code>ContentChannel</code>. - </li> -</ul> - -<p><code>RequestContext</code> keeps track of a request and its input and output <code>ContentChannel</code>s, and - related objects. Since <code>RequestContext</code> is a <code>ResponseHandler</code>, it is responsible for - instantiating and returning a <code>ContentChannel</code> when an application calls <code>handleResponse()</code>. - Two types are supported, one that supports HTTP response chunking, and one that does not. The type used is chosen - automatically based on HTTP version, headers etc.</p> - -<p>Since the jDISC API is fully asynchronous, operations can occur in any order. This is very extensively tested in the - HTTP server implementation. For instance, an application (<code>RequestHandler</code>) may choose to respond and - close the output <code>ContentChannel</code> immediately upon receiving the request, before the body of the request - has been written into the input <code>ContentChannel</code> of the <code>RequestHandler</code>. All such cases are - tested and properly handled.</p> - -<p>As one can see from the illustration, <code>ChannelContext</code> is also a <code>Runnable</code>, i.e. it keeps one - thread per channel. The HTTP server has two modes of operation, <code>optimizeForPipeline</code> <code>true</code> - or <code>false</code> in <code>HttpServerConfig</code>.</p> - -<p>If <code>optimizeForPipeline</code> is set to <code>true</code>, response chunks are enqueued on a blocking queue in - <code>ChannelContext</code> when <code>ContentChannel.write()</code> is called. The <code>ChannelContext</code> - thread is responsible for actually writing them, and closing the channel when appropriate. Since the HTTP server - supports pipelining, and writes from an application may occur in any order, special care is taken to write response - chunks in the correct order.</p> - -<p>If <code>optimizeForPipeline</code> is set to <code>false</code>, a call to <code>ContentChannel.write()</code> will - lead to an actual write on the wire, iff. the given chunk to be written is the next in line. Otherwise this is a - no-op. This also means that a <code>ContentChannel.write()</code> may lead to a cascade of writes that have been - enqueued since they were out-of-order when their <code>write()</code> was called. The <code>ChannelContext</code> - thread still takes care of channel closing in most cases.</p> -</body> -</html> diff --git a/logd/src/apps/logd/.gitignore b/logd/src/apps/logd/.gitignore index cad0e581304..a9a724ce05c 100644 --- a/logd/src/apps/logd/.gitignore +++ b/logd/src/apps/logd/.gitignore @@ -1,3 +1,3 @@ /.depend /Makefile -/logd +/vespa-logd diff --git a/logd/src/apps/logd/CMakeLists.txt b/logd/src/apps/logd/CMakeLists.txt index 7caf8300617..24b39c45144 100644 --- a/logd/src/apps/logd/CMakeLists.txt +++ b/logd/src/apps/logd/CMakeLists.txt @@ -2,7 +2,7 @@ vespa_add_executable(logd_app SOURCES main.cpp - OUTPUT_NAME logd + OUTPUT_NAME vespa-logd INSTALL sbin DEPENDS logd diff --git a/logd/src/tests/rotate/rotate_test.sh b/logd/src/tests/rotate/rotate_test.sh index 2af4d92e078..053a6a3670c 100755 --- a/logd/src/tests/rotate/rotate_test.sh +++ b/logd/src/tests/rotate/rotate_test.sh @@ -18,7 +18,7 @@ sleep 5 $SOURCE_DIRECTORY/create_configfile.sh export VESPA_CONFIG_ID=file:logd.cfg -../../apps/logd/logd & +../../apps/logd/vespa-logd & echo $! > logd.pid ./logd_dummylogger_app diff --git a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/HostApi.java b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/HostApi.java index b389ac37aa3..e5ff92011da 100644 --- a/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/HostApi.java +++ b/orchestrator-restapi/src/main/java/com/yahoo/vespa/orchestrator/restapi/HostApi.java @@ -1,6 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator.restapi; +import com.yahoo.vespa.jaxrs.annotation.PATCH; import com.yahoo.vespa.orchestrator.restapi.wire.GetHostResponse; import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostRequest; import com.yahoo.vespa.orchestrator.restapi.wire.PatchHostResponse; @@ -41,9 +42,8 @@ public interface HostApi { /** * Tweak internal Orchestrator state for host. - * Note: This should really be a patch method, but I was unable to get that working with PATCH from jaxrs_utils!? */ - @PUT + @PATCH @Path("/{hostname}") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) diff --git a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java index 54f1723c914..916ab9d3e5e 100644 --- a/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java +++ b/orchestrator/src/main/java/com/yahoo/vespa/orchestrator/resources/HostResource.java @@ -21,7 +21,6 @@ import javax.ws.rs.BadRequestException; import javax.ws.rs.InternalServerErrorException; import javax.ws.rs.NotFoundException; import javax.ws.rs.Path; -import javax.ws.rs.PathParam; import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -54,7 +53,7 @@ public class HostResource implements HostApi { } @Override - public PatchHostResponse patch(@PathParam("hostname") String hostNameString, PatchHostRequest request) { + public PatchHostResponse patch(String hostNameString, PatchHostRequest request) { HostName hostName = new HostName(hostNameString); if (request.state != null) { diff --git a/persistence/src/tests/proxy/.gitignore b/persistence/src/tests/proxy/.gitignore index 9bd2934723e..03bce028dd9 100644 --- a/persistence/src/tests/proxy/.gitignore +++ b/persistence/src/tests/proxy/.gitignore @@ -3,7 +3,7 @@ /providerstub_test /providerproxy_test /providerproxy_conformancetest -/external_providerproxy_conformancetest +/vespa-external-providerproxy-conformancetest persistence_providerproxy_conformance_test_app persistence_providerproxy_test_app persistence_providerstub_test_app diff --git a/persistence/src/tests/proxy/CMakeLists.txt b/persistence/src/tests/proxy/CMakeLists.txt index 14d19f6d4c6..f727773f564 100644 --- a/persistence/src/tests/proxy/CMakeLists.txt +++ b/persistence/src/tests/proxy/CMakeLists.txt @@ -21,7 +21,7 @@ vespa_add_executable(persistence_providerproxy_conformance_test_app TEST vespa_add_executable(persistence_external_providerproxy_conformancetest_app SOURCES external_providerproxy_conformancetest.cpp - OUTPUT_NAME external_providerproxy_conformancetest + OUTPUT_NAME vespa-external-providerproxy-conformancetest INSTALL bin DEPENDS persistence diff --git a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp index 80b78d420ec..dab4dcf1dff 100644 --- a/searchcore/src/tests/proton/summaryengine/summaryengine.cpp +++ b/searchcore/src/tests/proton/summaryengine/summaryengine.cpp @@ -381,7 +381,7 @@ verifyReply(size_t count, document::CompressionConfig::Type encoding, size_t org DataBuffer uncompressed; ConstBufferRef blob(ret[2]._data._buf, ret[2]._data._len); - document::decompress(CompressionConfig::toType(ret[0]._intval8), ret[1]._intval32, blob, uncompressed, false); + compression::decompress(CompressionConfig::toType(ret[0]._intval8), ret[1]._intval32, blob, uncompressed, false); EXPECT_EQUAL(orgSize, uncompressed.getDataLen()); vespalib::Slime summaries; @@ -402,7 +402,7 @@ verifyRPC(size_t count, CompressionConfig config(requestCompression, 9, 100); DataBuffer compressed(const_cast<char *>(buf.get().data), buf.get().size); - CompressionConfig::Type type = document::compress(config, ConstBufferRef(buf.get().data, buf.get().size), compressed, true); + CompressionConfig::Type type = compression::compress(config, ConstBufferRef(buf.get().data, buf.get().size), compressed, true); EXPECT_EQUAL(type, requestCompression); FRT_RPCRequest * request = new FRT_RPCRequest(); diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index c851d124970..8c4e7b4571a 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -210,21 +210,21 @@ summary.cache.allowvisitcaching bool default=false restart summary.cache.initialentries long default=0 restart ## Control compression type of the summary while in the cache. -summary.cache.compression.type enum {NONE, LZ4} default=LZ4 restart +summary.cache.compression.type enum {NONE, LZ4, ZSTD} default=LZ4 restart ## Control compression level of the summary while in cache. summary.cache.compression.level int default=9 restart ## Control compression type of the summary while in memory during compaction ## NB So far only stragey=LOG honours it. -summary.log.compact.compression.type enum {NONE, LZ4} default=LZ4 restart +summary.log.compact.compression.type enum {NONE, LZ4, ZSTD} default=LZ4 restart ## Control compression level of the summary while in memory during compaction summary.log.compact.compression.level int default=9 restart ## Control compression type of the summary ## NB So far only stragey=LOG honours it. -summary.log.chunk.compression.type enum {NONE, LZ4} default=LZ4 restart +summary.log.chunk.compression.type enum {NONE, LZ4, ZSTD} default=LZ4 restart ## Control compression level of the summary summary.log.chunk.compression.level int default=9 restart diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 0253e943883..e5a6d6aed22 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -117,6 +117,8 @@ deriveCompression(const T & config) { document::CompressionConfig compression; if (config.type == T::LZ4) { compression.type = document::CompressionConfig::LZ4; + } else if (config.type == T::ZSTD) { + compression.type = document::CompressionConfig::ZSTD; } compression.compressionLevel = config.level; return compression; @@ -160,6 +162,8 @@ SummaryManager::SummaryManager(vespalib::ThreadExecutor & executor, summary.compact2buckets ? bucketizer : search::IBucketizer::SP())); } +SummaryManager::~SummaryManager() {} + void SummaryManager::putDocument(uint64_t syncToken, const Document & doc, search::DocumentIdT lid) { diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h index b55345ae470..a1bcd34fd0f 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h @@ -82,6 +82,7 @@ public: const search::common::FileHeaderContext &fileHeaderContext, search::transactionlog::SyncProxy &tlSyncer, const std::shared_ptr<search::IBucketizer> & bucketizer); + ~SummaryManager(); void putDocument(uint64_t syncToken, const document::Document & doc, search::DocumentIdT lid); diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp index 99b39f68006..465c6fa9d03 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/docsum_by_slime.cpp @@ -4,9 +4,9 @@ #include <vespa/searchlib/util/slime_output_raw_buf_adapter.h> #include <vespa/searchlib/common/packets.h> #include <vespa/fnet/frt/rpcrequest.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/log/log.h> - LOG_SETUP(".proton.summaryengine.docsum_by_slime"); namespace proton { @@ -90,12 +90,14 @@ DocsumByRPC::DocsumByRPC(DocsumBySlime & slimeDocsumServer) : void DocsumByRPC::getDocsums(FRT_RPCRequest & req) { + using document::compression::decompress; + using document::compression::compress; FRT_Values &arg = *req.GetParams(); uint8_t encoding = arg[0]._intval8; uint32_t uncompressedSize = arg[1]._intval32; DataBuffer uncompressed(arg[2]._data._buf, arg[2]._data._len); ConstBufferRef blob(arg[2]._data._buf, arg[2]._data._len); - document::decompress(CompressionConfig::toType(encoding), uncompressedSize, blob, uncompressed, true); + decompress(CompressionConfig::toType(encoding), uncompressedSize, blob, uncompressed, true); assert(uncompressedSize == uncompressed.getDataLen()); vespalib::Slime summariesToGet; BinaryFormat::decode(Memory(uncompressed.getData(), uncompressed.getDataLen()), summariesToGet); @@ -108,7 +110,7 @@ DocsumByRPC::getDocsums(FRT_RPCRequest & req) BinaryFormat::encode(*summaries, output); ConstBufferRef buf(rbuf.GetDrainPos(), rbuf.GetUsedLen()); DataBuffer compressed(rbuf.GetWritableDrainPos(0), rbuf.GetUsedLen()); - CompressionConfig::Type type = document::compress(getCompressionConfig(), buf, compressed, true); + CompressionConfig::Type type = compress(getCompressionConfig(), buf, compressed, true); FRT_Values &ret = *req.GetReturn(); ret.AddInt8(type); diff --git a/searchlib/src/tests/docstore/chunk/chunk_test.cpp b/searchlib/src/tests/docstore/chunk/chunk_test.cpp index 8b08fb278e3..9be11dba84b 100644 --- a/searchlib/src/tests/docstore/chunk/chunk_test.cpp +++ b/searchlib/src/tests/docstore/chunk/chunk_test.cpp @@ -68,4 +68,34 @@ TEST("require that Chunk formats does not change between releases") testChunkFormat(v2, 34, "34 015BA32DE7000000220000000010ABCDEF987654321000000000000000074D000694"); } +constexpr const char * MY_LONG_STRING = "This is medium long string that hopefully will compress to something where lz4, zstandard and none" +" will make a difference. The intentions is to verify that we trigger all compresssions possible and are able to decompress them too." +" I guess that we need a considerable length in order to get the rather inefficient lz4 compression triger. ZStandard compression" +" should trigger a lot earlier"; + +void verifyChunkCompression(CompressionConfig::Type cfgType, const void * buf, size_t sz, size_t expectedLen) { + uint64_t MAGIC_CONTENT(0xabcdef9876543210); + ChunkFormatV2 chunk(10); + chunk.getBuffer() << MAGIC_CONTENT; + chunk.getBuffer().write(buf, sz); + vespalib::DataBuffer buffer; + CompressionConfig cfg(cfgType); + chunk.pack(7, buffer, cfg); + EXPECT_EQUAL(expectedLen, buffer.getDataLen()); + vespalib::nbostream is(buffer.getData(), buffer.getDataLen()); + ChunkFormat::UP deserialized = ChunkFormat::deserialize(buffer.getData(), buffer.getDataLen(), false); + uint64_t magic(0); + deserialized->getBuffer() >> magic; + EXPECT_EQUAL(MAGIC_CONTENT, magic); + std::vector<char> v(sz); + deserialized->getBuffer().read(&v[0], sz); + EXPECT_EQUAL(0, memcmp(buf, &v[0], sz)); +} + +TEST("require that V2 can create and handle lz4, zstd, and none") { + verifyChunkCompression(CompressionConfig::NONE, MY_LONG_STRING, strlen(MY_LONG_STRING), 421); + verifyChunkCompression(CompressionConfig::LZ4, MY_LONG_STRING, strlen(MY_LONG_STRING), 360); + verifyChunkCompression(CompressionConfig::ZSTD, MY_LONG_STRING, strlen(MY_LONG_STRING), 282); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/common/packets.cpp b/searchlib/src/vespa/searchlib/common/packets.cpp index 531305789b3..d885442ec03 100644 --- a/searchlib/src/vespa/searchlib/common/packets.cpp +++ b/searchlib/src/vespa/searchlib/common/packets.cpp @@ -7,6 +7,7 @@ #include <vespa/document/util/compressor.h> #include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/data/databuffer.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.common.fs4packets"); @@ -150,7 +151,7 @@ FS4PersistentPacketStreamer::Decode(FNET_DataBuffer *src, uint32_t plen, uint32_ uint32_t uncompressed_size = src->ReadInt32(); ConstBufferRef org(src->GetData(), plen - sizeof(uint32_t)); vespalib::DataBuffer uncompressed(uncompressed_size); - document::decompress(compressionType, uncompressed_size, org, uncompressed, false); + document::compression::decompress(compressionType, uncompressed_size, org, uncompressed, false); FNET_DataBuffer buf(uncompressed.getData(), uncompressed.getDataLen()); decodePacket(packet, buf, uncompressed_size, pcode); src->DataToDead(plen - sizeof(uint32_t)); @@ -191,7 +192,7 @@ FS4PersistentPacketStreamer::Encode(FNET_Packet *packet, uint32_t chid, FNET_Dat CompressionConfig config(_compressionType, _compressionLevel, 90); ConstBufferRef org(dst->GetData() + packet_start + header_len, body_len); vespalib::DataBuffer compressed(org.size()); - CompressionConfig::Type r = document::compress(config, org, compressed, false); + CompressionConfig::Type r = document::compression::compress(config, org, compressed, false); if (r != CompressionConfig::NONE) { dst->DataToFree(body_len + header_len); // sizeof(data + header + uncompressed_size) - sizeof(uint32_t) @@ -454,7 +455,7 @@ FS4Packet_PreSerialized::FS4Packet_PreSerialized(FNET_Packet & packet) 90); ConstBufferRef org(tmp.GetData(), tmp.GetDataLen()); vespalib::DataBuffer compressed(org.size()); - _compressionType = document::compress(config, org, compressed, false); + _compressionType = document::compression::compress(config, org, compressed, false); if (_compressionType != CompressionConfig::NONE) { _data.WriteInt32Fast(body_len); _data.WriteBytes(compressed.getData(), compressed.getDataLen()); diff --git a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp index cddf8c96b2e..4d4d4c48130 100644 --- a/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp +++ b/searchlib/src/vespa/searchlib/docstore/chunkformat.cpp @@ -8,6 +8,10 @@ namespace search { using vespalib::make_string; using vespalib::Exception; +using document::compression::compress; +using document::compression::decompress; +using document::compression::computeMaxCompressedsize; +using document::CompressionConfig; ChunkException::ChunkException(const vespalib::stringref & msg, const vespalib::stringref & location) : Exception(make_string("Illegal chunk: %s", msg.c_str()), location) @@ -15,7 +19,7 @@ ChunkException::ChunkException(const vespalib::stringref & msg, const vespalib:: } void -ChunkFormat::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const document::CompressionConfig & compression) +ChunkFormat::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const CompressionConfig & compression) { vespalib::nbostream & os = _dataBuf; os << lastSerial; @@ -29,7 +33,7 @@ ChunkFormat::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const const size_t oldPos(compressed.getDataLen()); compressed.writeInt8(compression.type); compressed.writeInt32(os.size()); - document::CompressionConfig::Type type(document::compress(compression, vespalib::ConstBufferRef(os.c_str(), os.size()), compressed, false)); + CompressionConfig::Type type(compress(compression, vespalib::ConstBufferRef(os.c_str(), os.size()), compressed, false)); if (compression.type != type) { compressed.getData()[oldPos] = type; } @@ -42,28 +46,22 @@ ChunkFormat::pack(uint64_t lastSerial, vespalib::DataBuffer & compressed, const } size_t -ChunkFormat::getMaxPackSize(const document::CompressionConfig & compression) const +ChunkFormat::getMaxPackSize(const CompressionConfig & compression) const { const size_t OVERHEAD(0); const size_t MINSIZE(1 + 1 + 4 + 4 + includeSerializedSize() ? 4 : 0); // version + type + real length + crc + lastserial const size_t formatSpecificSize(getHeaderSize()); size_t rawSize(MINSIZE + formatSpecificSize + OVERHEAD); const size_t payloadSize(_dataBuf.size() + 8); - // This is a little dirty -> need interface. - if (compression.type == document::CompressionConfig::LZ4) { - document::LZ4Compressor lz4; - rawSize += lz4.adjustProcessLen(0, payloadSize); - } else { - rawSize += payloadSize; - } - return rawSize; + return rawSize + computeMaxCompressedsize(compression.type, payloadSize); } void ChunkFormat::verifyCompression(uint8_t type) { - if ((type != document::CompressionConfig::LZ4) && - (type != document::CompressionConfig::NONE)) { + if ((type != CompressionConfig::LZ4) && + (type != CompressionConfig::ZSTD) && + (type != CompressionConfig::NONE)) { throw ChunkException(make_string("Unknown compressiontype %d", type), VESPA_STRLOC); } } @@ -145,7 +143,7 @@ ChunkFormat::deserializeBody(vespalib::nbostream & is) // This is a dirty trick to fool some odd sanity checking in DataBuffer::swap vespalib::DataBuffer uncompressed(const_cast<char *>(is.peek()), (size_t)0); vespalib::ConstBufferRef data(is.peek(), is.size() - sizeof(uint32_t)); - document::decompress(document::CompressionConfig::Type(type), uncompressedLen, data, uncompressed, true); + decompress(CompressionConfig::Type(type), uncompressedLen, data, uncompressed, true); assert(uncompressed.getData() == uncompressed.getDead()); if (uncompressed.getData() != data.c_str()) { const size_t sz(uncompressed.getDataLen()); diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp index fd7df29cd49..ffb56407198 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.cpp @@ -8,6 +8,11 @@ #include <vespa/vespalib/data/databuffer.h> #include <vespa/document/util/compressor.h> +using document::DocumentTypeRepo; +using document::CompressionConfig; +using document::compression::compress; +using document::compression::decompress; + namespace search { namespace { @@ -15,13 +20,13 @@ namespace { class DocumentVisitorAdapter : public IBufferVisitor { public: - DocumentVisitorAdapter(const document::DocumentTypeRepo & repo, IDocumentVisitor & visitor) : + DocumentVisitorAdapter(const DocumentTypeRepo & repo, IDocumentVisitor & visitor) : _repo(repo), _visitor(visitor) { } void visit(uint32_t lid, vespalib::ConstBufferRef buf) override; private: - const document::DocumentTypeRepo & _repo; + const DocumentTypeRepo & _repo; IDocumentVisitor & _visitor; }; @@ -45,7 +50,7 @@ public: using Alloc = vespalib::alloc::Alloc; typedef std::unique_ptr<Value> UP; - Value() : _compressedSize(0), _uncompressedSize(0), _compression(document::CompressionConfig::NONE) {} + Value() : _compressedSize(0), _uncompressedSize(0), _compression(CompressionConfig::NONE) {} Value(Value &&rhs) : _compressedSize(rhs._compressedSize), @@ -69,12 +74,12 @@ public: return *this; } - void setCompression(document::CompressionConfig::Type comp, size_t uncompressedSize) { + void setCompression(CompressionConfig::Type comp, size_t uncompressedSize) { _compression = comp; _uncompressedSize = uncompressedSize; } - document::CompressionConfig::Type getCompression() const { return _compression; } + CompressionConfig::Type getCompression() const { return _compression; } size_t getUncompressedSize() const { return _uncompressedSize; } @@ -82,13 +87,13 @@ public: * Compress buffer into temporary buffer and copy temporary buffer to * value along with compression config. */ - void set(vespalib::DataBuffer &&buf, ssize_t len, const document::CompressionConfig &compression); + void set(vespalib::DataBuffer &&buf, ssize_t len, const CompressionConfig &compression); /** * Decompress value into temporary buffer and deserialize document from * the temporary buffer. */ - document::Document::UP deserializeDocument(const document::DocumentTypeRepo &repo); + document::Document::UP deserializeDocument(const DocumentTypeRepo &repo); size_t size() const { return _compressedSize; } bool empty() const { return size() == 0; } @@ -98,36 +103,32 @@ public: private: size_t _compressedSize; size_t _uncompressedSize; - document::CompressionConfig::Type _compression; + CompressionConfig::Type _compression; Alloc _buf; }; class BackingStore { public: - BackingStore(IDataStore &store, const document::CompressionConfig &compression) : + BackingStore(IDataStore &store, const CompressionConfig &compression) : _backingStore(store), _compression(compression) { } bool read(DocumentIdT key, Value &value) const; - void visit(const IDocumentStore::LidVector &lids, const document::DocumentTypeRepo &repo, IDocumentVisitor &visitor) const; + void visit(const IDocumentStore::LidVector &lids, const DocumentTypeRepo &repo, IDocumentVisitor &visitor) const; void write(DocumentIdT, const Value &) {} void erase(DocumentIdT) {} - const document::CompressionConfig &getCompression(void) const { return _compression; } + const CompressionConfig &getCompression(void) const { return _compression; } private: IDataStore &_backingStore; - const document::CompressionConfig _compression; + const CompressionConfig _compression; }; void -Value::set(vespalib::DataBuffer &&buf, - ssize_t len, - const document::CompressionConfig &compression) { +Value::set(vespalib::DataBuffer &&buf, ssize_t len, const CompressionConfig &compression) { //Underlying buffer must be identical to allow swap. vespalib::DataBuffer compressed(buf.getData(), 0u); - document::CompressionConfig::Type type = - document::compress(compression, - vespalib::ConstBufferRef(buf.getData(), len), - compressed, true); + CompressionConfig::Type type = compress(compression, vespalib::ConstBufferRef(buf.getData(), len), + compressed, true); _compressedSize = compressed.getDataLen(); if (buf.getData() == compressed.getData()) { // Uncompressed so we can just steal the underlying buffer. @@ -135,28 +136,25 @@ Value::set(vespalib::DataBuffer &&buf, } else { compressed.stealBuffer().swap(_buf); } - assert(((type == document::CompressionConfig::NONE) && + assert(((type == CompressionConfig::NONE) && (len == ssize_t(_compressedSize))) || - ((type != document::CompressionConfig::NONE) && + ((type != CompressionConfig::NONE) && (len > ssize_t(_compressedSize)))); setCompression(type, len); } document::Document::UP -Value::deserializeDocument(const document::DocumentTypeRepo &repo) { +Value::deserializeDocument(const DocumentTypeRepo &repo) { vespalib::DataBuffer uncompressed((char *) _buf.get(), (size_t) 0); - document::decompress(getCompression(), - getUncompressedSize(), - vespalib::ConstBufferRef(*this, size()), - uncompressed, true); + decompress(getCompression(), getUncompressedSize(), vespalib::ConstBufferRef(*this, size()), uncompressed, true); vespalib::nbostream is(uncompressed.getData(), uncompressed.getDataLen()); return document::Document::UP(new document::Document(repo, is)); } void -BackingStore::visit(const IDocumentStore::LidVector &lids, const document::DocumentTypeRepo &repo, +BackingStore::visit(const IDocumentStore::LidVector &lids, const DocumentTypeRepo &repo, IDocumentVisitor &visitor) const { DocumentVisitorAdapter adapter(repo, visitor); _backingStore.read(lids, adapter); @@ -213,7 +211,7 @@ DocumentStore::useCache() const { } void -DocumentStore::visit(const LidVector & lids, const document::DocumentTypeRepo &repo, IDocumentVisitor & visitor) const +DocumentStore::visit(const LidVector & lids, const DocumentTypeRepo &repo, IDocumentVisitor & visitor) const { if (useCache() && _config.allowVisitCaching() && visitor.allowVisitCaching()) { docstore::BlobSet blobSet = _visitCache->read(lids).getBlobSet(); @@ -227,7 +225,7 @@ DocumentStore::visit(const LidVector & lids, const document::DocumentTypeRepo &r } document::Document::UP -DocumentStore::read(DocumentIdT lid, const document::DocumentTypeRepo &repo) const +DocumentStore::read(DocumentIdT lid, const DocumentTypeRepo &repo) const { document::Document::UP retval; Value value; @@ -305,25 +303,23 @@ DocumentStore::getLastFlushTime() const template <class Visitor> class DocumentStore::WrapVisitor : public IDataStoreVisitor { - Visitor &_visitor; - const document::DocumentTypeRepo &_repo; - const document::CompressionConfig &_compression; - IDocumentStore &_ds; - uint64_t _syncToken; + Visitor &_visitor; + const DocumentTypeRepo &_repo; + const CompressionConfig &_compression; + IDocumentStore &_ds; + uint64_t _syncToken; public: void visit(uint32_t lid, const void *buffer, size_t sz) override; WrapVisitor(Visitor &visitor, - const document::DocumentTypeRepo &repo, - const document::CompressionConfig &compresion, + const DocumentTypeRepo &repo, + const CompressionConfig &compresion, IDocumentStore &ds, uint64_t syncToken); inline void rewrite(uint32_t lid, const document::Document &doc); - inline void rewrite(uint32_t lid); - inline void visitRemove(uint32_t lid); }; @@ -429,8 +425,8 @@ DocumentStore::WrapVisitor<Visitor>::visit(uint32_t lid, template <class Visitor> DocumentStore::WrapVisitor<Visitor>:: WrapVisitor(Visitor &visitor, - const document::DocumentTypeRepo &repo, - const document::CompressionConfig &compression, + const DocumentTypeRepo &repo, + const CompressionConfig &compression, IDocumentStore &ds, uint64_t syncToken) : _visitor(visitor), @@ -445,7 +441,7 @@ WrapVisitor(Visitor &visitor, void DocumentStore::accept(IDocumentStoreReadVisitor &visitor, IDocumentStoreVisitorProgress &visitorProgress, - const document::DocumentTypeRepo &repo) + const DocumentTypeRepo &repo) { WrapVisitor<IDocumentStoreReadVisitor> wrap(visitor, repo, _store->getCompression(), @@ -460,7 +456,7 @@ DocumentStore::accept(IDocumentStoreReadVisitor &visitor, void DocumentStore::accept(IDocumentStoreRewriteVisitor &visitor, IDocumentStoreVisitorProgress &visitorProgress, - const document::DocumentTypeRepo &repo) + const DocumentTypeRepo &repo) { WrapVisitor<IDocumentStoreRewriteVisitor> wrap(visitor, repo, diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index 8fac288a23a..6771fb3bec6 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -90,7 +90,7 @@ CompressedBlobSet::CompressedBlobSet(const document::CompressionConfig &compress if ( ! _positions.empty() ) { DataBuffer compressed; ConstBufferRef org = uncompressed.getBuffer(); - _compression = document::compress(compression, org, compressed, false); + _compression = document::compression::compress(compression, org, compressed, false); _buffer.resize(compressed.getDataLen()); memcpy(_buffer, compressed.getData(), compressed.getDataLen()); } @@ -99,10 +99,12 @@ CompressedBlobSet::CompressedBlobSet(const document::CompressionConfig &compress BlobSet CompressedBlobSet::getBlobSet() const { + using document::compression::decompress; // These are frequent lage allocations that are to expensive to mmap. DataBuffer uncompressed(0, 1, Alloc::alloc(0, 16 * MemoryAllocator::HUGEPAGE_SIZE)); if ( ! _positions.empty() ) { - document::decompress(_compression, getBufferSize(_positions), ConstBufferRef(_buffer.c_str(), _buffer.size()), uncompressed, false); + decompress(_compression, getBufferSize(_positions), + ConstBufferRef(_buffer.c_str(), _buffer.size()), uncompressed, false); } return BlobSet(_positions, uncompressed.stealBuffer()); } diff --git a/searchlib/src/vespa/searchlib/grouping/sketch.h b/searchlib/src/vespa/searchlib/grouping/sketch.h index 50209b19bbd..a036d23ef36 100644 --- a/searchlib/src/vespa/searchlib/grouping/sketch.h +++ b/searchlib/src/vespa/searchlib/grouping/sketch.h @@ -209,7 +209,7 @@ compress_buckets_into(char *buffer, uint32_t size) const { vespalib::ConstBufferRef org(&bucket[0], BUCKET_COUNT); vespalib::DataBuffer compress_buffer(buffer, size); document::CompressionConfig::Type r = - document::compress(config, org, compress_buffer, false); + document::compression::compress(config, org, compress_buffer, false); assert(compress_buffer.getDead() == buffer); if (r == document::CompressionConfig::LZ4) { assert(compress_buffer.getDataLen() < BUCKET_COUNT); @@ -228,7 +228,7 @@ decompress_buckets_from(char *buffer, uint32_t size) { } else { vespalib::ConstBufferRef compressed(buffer, size); vespalib::DataBuffer uncompressed(reinterpret_cast<char *>(&bucket[0]), BUCKET_COUNT); - document::decompress(document::CompressionConfig::LZ4, BUCKET_COUNT, compressed, uncompressed, false); + document::compression::decompress(document::CompressionConfig::LZ4, BUCKET_COUNT, compressed, uncompressed, false); } } template <int BucketBits, typename HashT> diff --git a/staging_vespalib/src/vespa/vespalib/data/databuffer.h b/staging_vespalib/src/vespa/vespalib/data/databuffer.h index a9ed53e2f84..875aa2f0074 100644 --- a/staging_vespalib/src/vespa/vespalib/data/databuffer.h +++ b/staging_vespalib/src/vespa/vespalib/data/databuffer.h @@ -1,10 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <string.h> -#include <stdint.h> -#include <stdlib.h> -#include <assert.h> +#include <cstring> +#include <cassert> #include <vespa/vespalib/util/alloc.h> namespace vespalib { diff --git a/vespabase/CMakeLists.txt b/vespabase/CMakeLists.txt index 9c0aced2319..cbd7bd3d334 100644 --- a/vespabase/CMakeLists.txt +++ b/vespabase/CMakeLists.txt @@ -11,7 +11,7 @@ vespa_install_script(src/start-cbinaries.sh vespa-transactionlog-inspect bin) vespa_install_script(src/start-cbinaries.sh vespa-vds-disktool bin) vespa_install_script(src/start-cbinaries.sh vespa-distributord sbin) vespa_install_script(src/start-cbinaries.sh vespa-fdispatch sbin) -vespa_install_script(src/start-cbinaries.sh vespa-filedistributor sbin) +vespa_install_script(src/start-cbinaries.sh filedistributor sbin) vespa_install_script(src/start-cbinaries.sh vespa-proton sbin) vespa_install_script(src/start-cbinaries.sh vespa-storaged sbin) diff --git a/vespalib/src/vespa/vespalib/util/optimized.h b/vespalib/src/vespa/vespalib/util/optimized.h index ec11bbb686a..c5d0404e310 100644 --- a/vespalib/src/vespa/vespalib/util/optimized.h +++ b/vespalib/src/vespa/vespalib/util/optimized.h @@ -4,7 +4,7 @@ #pragma once -#include <stdint.h> +#include <cstdint> namespace vespalib { diff --git a/vespalog/src/vespa/log/bufferedlogger.cpp b/vespalog/src/vespa/log/bufferedlogger.cpp index db2021324c3..fe6a03ebf69 100644 --- a/vespalog/src/vespa/log/bufferedlogger.cpp +++ b/vespalog/src/vespa/log/bufferedlogger.cpp @@ -10,11 +10,9 @@ #include <boost/multi_index/sequenced_index.hpp> #include <iomanip> -#include <iostream> #include <sstream> #include <vector> #include <cstdarg> -#include <ctime> namespace ns_log { @@ -381,5 +379,4 @@ BufferedLogger::setTimer(std::unique_ptr<Timer> timer) _backing->_timer = std::move(timer); } - } // ns_log diff --git a/vespamalloc/CMakeLists.txt b/vespamalloc/CMakeLists.txt index 11bc1b60983..ec4293979f8 100644 --- a/vespamalloc/CMakeLists.txt +++ b/vespamalloc/CMakeLists.txt @@ -24,5 +24,5 @@ vespa_define_module( src/vespamalloc/util ) -vespa_install_script(bin/parsememorydump.pl vespamalloc-parsememorydump.pl bin) +vespa_install_script(bin/parsememorydump.pl vespa-malloc-parse-memorydump.pl bin) install(FILES etc/vespamalloc.conf PERMISSIONS OWNER_READ GROUP_READ WORLD_READ DESTINATION etc) |