diff options
26 files changed, 118 insertions, 311 deletions
diff --git a/build_settings.cmake b/build_settings.cmake index 2edf48103cf..ac4e86c2e81 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -83,6 +83,8 @@ else() endif() SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic" ) +message("-- CMAKE_SHARED_LINKER_FLAGS is ${CMAKE_SHARED_LINKER_FLAGS}") + # Use C++ 17 # TODO renable when cmake 3.8 is out. # set(CMAKE_CXX_STANDARD 17) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 8ebb449ceda..c406fa80434 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -85,7 +85,7 @@ public interface ModelContext { // TODO: Remove after April 2020 default boolean usePhraseSegmenting() { return false; } - default String proxyProtocol() { return "https-only"; } + default String proxyProtocol() { return "https+proxy-protocol"; } // TODO bjorncs: Remove after end of May default Optional<AthenzDomain> athenzDomain() { return Optional.empty(); } // TODO(mpolden): Remove after May 2020 diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index fb8e9dffbbb..bcc2c9a3d6a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -21,36 +21,33 @@ public class HostedSslConnectorFactory extends ConnectorFactory { private static final String DEFAULT_HOSTED_TRUSTSTORE = "/opt/yahoo/share/ssl/certs/athenz_certificate_bundle.pem"; private final boolean enforceClientAuth; - private final String proxyProtocol; /** * Create connector factory that uses a certificate provided by the config-model / configserver and default hosted Vespa truststore. */ // TODO Enforce client authentication - public static HostedSslConnectorFactory withProvidedCertificate(String proxyProtocol, String serverName, EndpointCertificateSecrets endpointCertificateSecrets) { - return new HostedSslConnectorFactory(proxyProtocol, - createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null), false); + public static HostedSslConnectorFactory withProvidedCertificate( + String serverName, EndpointCertificateSecrets endpointCertificateSecrets) { + return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, DEFAULT_HOSTED_TRUSTSTORE, /*tlsCaCertificates*/null), false); } /** * Create connector factory that uses a certificate provided by the config-model / configserver and a truststore configured by the application. */ public static HostedSslConnectorFactory withProvidedCertificateAndTruststore( - String proxyProtocol, String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) { - return new HostedSslConnectorFactory(proxyProtocol, - createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates), true); + String serverName, EndpointCertificateSecrets endpointCertificateSecrets, String tlsCaCertificates) { + return new HostedSslConnectorFactory(createConfiguredDirectSslProvider(serverName, endpointCertificateSecrets, /*tlsCaCertificatesPath*/null, tlsCaCertificates), true); } /** * Create connector factory that uses the default certificate and truststore provided by Vespa (through Vespa-global TLS configuration). */ - public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String proxyProtocol, String serverName) { - return new HostedSslConnectorFactory(proxyProtocol, new DefaultSslProvider(serverName), true); + public static HostedSslConnectorFactory withDefaultCertificateAndTruststore(String serverName) { + return new HostedSslConnectorFactory(new DefaultSslProvider(serverName), true); } - private HostedSslConnectorFactory(String proxyProtocol, SimpleComponent sslProviderComponent, boolean enforceClientAuth) { + private HostedSslConnectorFactory(SimpleComponent sslProviderComponent, boolean enforceClientAuth) { super("tls4443", 4443, sslProviderComponent); - this.proxyProtocol = proxyProtocol; this.enforceClientAuth = enforceClientAuth; } @@ -70,25 +67,10 @@ public class HostedSslConnectorFactory extends ConnectorFactory { super.getConfig(connectorBuilder); connectorBuilder .tlsClientAuthEnforcer(new ConnectorConfig.TlsClientAuthEnforcer.Builder() - .pathWhitelist(INSECURE_WHITELISTED_PATHS) - .enable(enforceClientAuth)) - .proxyProtocol(configureProxyProtocol()) + .pathWhitelist(INSECURE_WHITELISTED_PATHS) + .enable(enforceClientAuth)) + .proxyProtocol(new ConnectorConfig.ProxyProtocol.Builder().enabled(true).mixedMode(true)) .idleTimeout(Duration.ofMinutes(3).toSeconds()) .maxConnectionLife(Duration.ofMinutes(10).toSeconds()); } - - private ConnectorConfig.ProxyProtocol.Builder configureProxyProtocol() { - ConnectorConfig.ProxyProtocol.Builder proxyProtocolBuilder = new ConnectorConfig.ProxyProtocol.Builder(); - switch (proxyProtocol) { - case "https-only": - return proxyProtocolBuilder.enabled(false).mixedMode(false); - case "https+proxy-protocol": - return proxyProtocolBuilder.enabled(true).mixedMode(true); - case "proxy-protocol-only": - return proxyProtocolBuilder.enabled(true).mixedMode(false); - default: - throw new IllegalArgumentException("Unknown proxy-protocol settings: " + proxyProtocol); - } - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 8c8e8be5a30..78ef475653d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -337,7 +337,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { JettyHttpServer server = cluster.getHttp().getHttpServer().get(); String serverName = server.getComponentId().getName(); - String proxyProtocol = deployState.getProperties().proxyProtocol(); // If the deployment contains certificate/private key reference, setup TLS port if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); @@ -346,11 +345,11 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get(); HostedSslConnectorFactory connectorFactory = authorizeClient - ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(proxyProtocol, serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get()) - : HostedSslConnectorFactory.withProvidedCertificate(proxyProtocol, serverName, endpointCertificateSecrets); + ? HostedSslConnectorFactory.withProvidedCertificateAndTruststore(serverName, endpointCertificateSecrets, deployState.tlsClientAuthority().get()) + : HostedSslConnectorFactory.withProvidedCertificate(serverName, endpointCertificateSecrets); server.addConnector(connectorFactory); } else { - server.addConnector(HostedSslConnectorFactory.withDefaultCertificateAndTruststore(proxyProtocol, serverName)); + server.addConnector(HostedSslConnectorFactory.withDefaultCertificateAndTruststore(serverName)); } } diff --git a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java index bc8369677fc..c8c30277953 100644 --- a/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java +++ b/configgen/src/main/java/com/yahoo/config/codegen/CppClassBuilder.java @@ -101,15 +101,8 @@ public class CppClassBuilder implements ClassBuilder { File headerFile = new File(rootDir, relativePathUnderRoot + "/" + getFileName(root, "h")); File bodyFile = new File(rootDir, relativePathUnderRoot + "/" + getFileName(root, "cpp")); - String oldHeader = readFile(headerFile); - String oldBody = readFile(bodyFile); - - if (oldHeader == null || !oldHeader.equals(newHeader)) { - writeFile(headerFile, newHeader); - } - if (oldBody == null || !oldBody.equals(newBody)) { - writeFile(bodyFile, newBody); - } + writeFile(headerFile, newHeader); + writeFile(bodyFile, newBody); } catch (IOException e) { e.printStackTrace(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 400460d2ce6..2773659559b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -150,7 +150,6 @@ public class ModelContextImpl implements ModelContext { private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets; private final double defaultTermwiseLimit; private final double defaultSoftStartSeconds; - private final String proxyProtocol; private final Optional<AthenzDomain> athenzDomain; public Properties(ApplicationId applicationId, @@ -187,8 +186,6 @@ public class ModelContextImpl implements ModelContext { .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); defaultTopKprobability = Flags.DEFAULT_TOP_K_PROBABILITY.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); - this.proxyProtocol = Flags.PROXY_PROTOCOL.bindTo(flagSource) - .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); this.athenzDomain = athenzDomain; } @@ -258,9 +255,6 @@ public class ModelContextImpl implements ModelContext { public boolean useNewAthenzFilter() { return true; } @Override - public String proxyProtocol() { return proxyProtocol; } - - @Override public Optional<AthenzDomain> athenzDomain() { return athenzDomain; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java index 820384ce810..9bd3b70cfab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java @@ -96,7 +96,8 @@ class SystemFlagsDeployer { createNewFlagData(target, dryRun, wantedFlagData, currentFlagData, results, errors); updateExistingFlagData(target, dryRun, wantedFlagData, currentFlagData, results, errors); removeOldFlagData(target, dryRun, wantedFlagData, currentFlagData, results, errors); - warnOnFlagDataForUndefinedFlags(target, wantedFlagData, currentFlagData, definedFlags, warnings); + failOnNewFlagDataForUndefinedFlags(target, wantedFlagData, currentFlagData, definedFlags, errors); + warnOnExistingFlagDataForUndefinedFlags(target, wantedFlagData, currentFlagData, definedFlags, warnings); return new SystemFlagsDeployResult(results, errors, warnings); } @@ -175,11 +176,23 @@ class SystemFlagsDeployer { }); } - private static void warnOnFlagDataForUndefinedFlags(FlagsTarget target, - Map<FlagId, FlagData> wantedFlagData, - Map<FlagId, FlagData> currentFlagData, - List<FlagId> definedFlags, - List<Warning> warnings) { + private static void failOnNewFlagDataForUndefinedFlags(FlagsTarget target, + Map<FlagId, FlagData> wantedFlagData, + Map<FlagId, FlagData> currentFlagData, + List<FlagId> definedFlags, + List<OperationError> errors) { + for (FlagId flagId : wantedFlagData.keySet()) { + if (!currentFlagData.containsKey(flagId) && !definedFlags.contains(flagId)) { + errors.add(OperationError.createFailed("Flag not defined in target zone", target, wantedFlagData.get(flagId))); + } + } + } + + private static void warnOnExistingFlagDataForUndefinedFlags(FlagsTarget target, + Map<FlagId, FlagData> wantedFlagData, + Map<FlagId, FlagData> currentFlagData, + List<FlagId> definedFlags, + List<Warning> warnings) { for (FlagId flagId : currentFlagData.keySet()) { if (wantedFlagData.containsKey(flagId) && !definedFlags.contains(flagId)) { warnings.add(Warning.dataForUndefinedFlag(target, flagId)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java index 475ac12f2fd..789be26db1f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java @@ -77,6 +77,7 @@ public class SystemFlagsDeployerTest { public void dryrun_should_not_change_flags() throws IOException { FlagsClient flagsClient = mock(FlagsClient.class); when(flagsClient.listFlagData(controllerTarget)).thenReturn(List.of()); + when(flagsClient.listDefinedFlags(controllerTarget)).thenReturn(List.of(new FlagId("my-flag"))); FlagData defaultData = flagData("flags/my-flag/main.json"); SystemFlagsDataArchive archive = new SystemFlagsDataArchive.Builder() @@ -101,6 +102,7 @@ public class SystemFlagsDeployerTest { UncheckedIOException exception = new UncheckedIOException(new IOException("I/O error message")); when(flagsClient.listFlagData(prodUsWest1Target)).thenThrow(exception); when(flagsClient.listFlagData(prodUsEast3Target)).thenReturn(List.of()); + when(flagsClient.listDefinedFlags(prodUsEast3Target)).thenReturn(List.of(new FlagId("my-flag"))); FlagData defaultData = flagData("flags/my-flag/main.json"); SystemFlagsDataArchive archive = new SystemFlagsDataArchive.Builder() @@ -133,6 +135,23 @@ public class SystemFlagsDeployerTest { } @Test + public void creates_error_entry_for_flag_data_of_undefined_flag() throws IOException { + FlagData prodUsEast3Data = flagData("flags/my-flag/main.prod.us-east-3.json"); + FlagsClient flagsClient = mock(FlagsClient.class); + when(flagsClient.listFlagData(prodUsEast3Target)) + .thenReturn(List.of()); + when(flagsClient.listDefinedFlags(prodUsEast3Target)) + .thenReturn(List.of()); + SystemFlagsDataArchive archive = new SystemFlagsDataArchive.Builder() + .addFile("main.prod.us-east-3.json", prodUsEast3Data) + .build(); + SystemFlagsDeployer deployer = new SystemFlagsDeployer(flagsClient, SYSTEM, Set.of(prodUsEast3Target)); + SystemFlagsDeployResult result = deployer.deployFlags(archive, true); + assertThat(result.errors()) + .containsOnly(SystemFlagsDeployResult.OperationError.createFailed("Flag not defined in target zone", prodUsEast3Target, prodUsEast3Data)); + } + + @Test public void creates_warning_entry_for_existing_flag_data_for_undefined_flag() throws IOException { FlagData prodUsEast3Data = flagData("flags/my-flag/main.prod.us-east-3.json"); FlagsClient flagsClient = mock(FlagsClient.class); diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 6ed2e06e060..c74a470ae43 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -5,6 +5,7 @@ include(VespaExtendedDefaultBuildSettings OPTIONAL) function(setup_vespa_default_build_settings_rhel_6_10) message("-- Setting up default build settings for rhel 6.10") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) + set(DEFAULT_CMAKE_SHARED_LINKER_FLAGS "-lrt" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_rhel_7_7) @@ -125,18 +126,13 @@ function(vespa_use_default_vespa_user) endfunction() function(vespa_use_default_build_settings) - if (DEFINED CMAKE_PREFIX_PATH AND - DEFINED VESPA_LLVM_VERSION AND - DEFINED EXTRA_INCLUDE_DIRECTORY AND DEFINED EXTRA_LINK_DIRECTORY AND - DEFINED CMAKE_INSTALL_RPATH AND DEFINED CMAKE_BUILD_RPATH) - return() - endif() set(VESPA_DEPS "/opt/vespa-deps") unset(DEFAULT_VESPA_LLVM_VERSION) unset(DEFAULT_CMAKE_PREFIX_PATH) unset(DEFAULT_EXTRA_LINK_DIRECTORY) unset(DEFAULT_EXTRA_INCLUDE_DIRECTORY) unset(DEFAULT_VESPA_CPU_ARCH_FLAGS) + unset(DEFAULT_CMAKE_SHARED_LINKER_FLAGS) if(APPLE) if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") set(VESPA_DEPS "/opt/vespa-deps-clang") @@ -190,6 +186,10 @@ function(vespa_use_default_build_settings) if(NOT DEFINED DEFAULT_EXTRA_INCLUDE_DIRECTORY) set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include") endif() + if(DEFINED DEFAULT_CMAKE_SHARED_LINKER_FLAGS) + message("-- DEFAULT_CMAKE_SHARED_LINKER_FLAGS is ${DEFAULT_CMAKE_SHARED_LINKER_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${DEFAULT_CMAKE_SHARED_LINKER_FLAGS}" PARENT_SCOPE) + endif() if(NOT DEFINED DEFAULT_VESPA_CPU_ARCH_FLAGS) if(VESPA_OS_DISTRO STREQUAL "fedora" AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-march=westmere -mtune=haswell") diff --git a/dist/vespa.spec b/dist/vespa.spec index b2f3d1d5df4..94ced88b9fb 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -118,6 +118,7 @@ Requires: epel-release %endif Requires: which Requires: initscripts +Requires: libcgroup-tools Requires: perl Requires: perl-Carp Requires: perl-Data-Dumper diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 40faefdd052..779d4e80056 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -232,12 +232,6 @@ public class Flags { "Takes effect on redeploy", ZONE_ID, APPLICATION_ID); - public static final UnboundStringFlag PROXY_PROTOCOL = defineStringFlag( - "proxy-protocol", "https-only", - "Enable proxy protocol support on application containers. Allowed values: ['https-only', 'https+proxy-protocol', 'proxy-protocol-only']", - "Takes effect on internal redeploy", - APPLICATION_ID); - public static final UnboundBooleanFlag ALLOW_DIRECT_ROUTING = defineFeatureFlag( "publish-direct-routing-endpoint", false, "Whether an application should receive a directly routed endpoint in its endpoint list", diff --git a/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp b/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp index 5f28f82c526..7cba56d5d72 100644 --- a/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp +++ b/searchlib/src/vespa/searchlib/common/tunefileinfo.hpp @@ -24,17 +24,21 @@ template <typename MMapConfig> void TuneFileRandRead::setFromMmapConfig(const MMapConfig & mmapFlags) { for (size_t i(0), m(mmapFlags.options.size()); i < m; i++) { +#ifdef __linux__ switch (mmapFlags.options[i]) { case MMapConfig::Options::MLOCK: _mmapFlags |= MAP_LOCKED; break; case MMapConfig::Options::POPULATE: _mmapFlags |= MAP_POPULATE; break; case MMapConfig::Options::HUGETLB: _mmapFlags |= MAP_HUGETLB; break; } +#endif } +#ifdef __linux__ switch (mmapFlags.advise) { case MMapConfig::Advise::NORMAL: setAdvise(POSIX_FADV_NORMAL); break; case MMapConfig::Advise::RANDOM: setAdvise(POSIX_FADV_RANDOM); break; case MMapConfig::Advise::SEQUENTIAL: setAdvise(POSIX_FADV_SEQUENTIAL); break; } +#endif } } diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp index f800166a647..b59405a73cd 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp @@ -144,7 +144,9 @@ GetOperation::onReceive(DistributorMessageSender& sender, const std::shared_ptr< LOG(debug, "Received %s", msg->toString(true).c_str()); - _msg->getTrace().getRoot().addChild(getreply->getTrace().getRoot()); + if ( ! getreply->getTrace().getRoot().isEmpty()) { + _msg->getTrace().getRoot().addChild(getreply->getTrace().getRoot()); + } bool allDone = true; for (auto& response : _responses) { for (uint32_t i = 0; i < response.second.size(); i++) { diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 4f49d89929f..2e6fe4020c2 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -643,7 +643,9 @@ TwoPhaseUpdateOperation::satisfiesUpdateTimestampConstraint(api::Timestamp ts) c void TwoPhaseUpdateOperation::addTraceFromReply(const api::StorageReply& reply) { - _trace.addChild(reply.getTrace().getRoot()); + if ( ! reply.getTrace().getRoot().isEmpty()) { + _trace.addChild(reply.getTrace().getRoot()); + } } void diff --git a/storage/src/vespa/storage/tools/.gitignore b/storage/src/vespa/storage/tools/.gitignore index 9f66a0b7157..26990358442 100644 --- a/storage/src/vespa/storage/tools/.gitignore +++ b/storage/src/vespa/storage/tools/.gitignore @@ -3,27 +3,10 @@ .*.swp .depend Makefile -analyzedistribution distbitreport.html -distributionsim -dumpslotfile generatedistributionbits -generatemailbuckets getidealstate idealstate -pingstorage -populatenode -slotfilefeeder -statfs -stoccart vespa-storage-cmd -throttlingsim -vdsclient -vdsdisktool -vdsidealstate -vesparemovelocation -storage_analyzedistribution_app storage_generatedistributionbits_app storage_getidealstate_app -storage_statfs_app -storage_throttlingsim_app diff --git a/storage/src/vespa/storage/tools/CMakeLists.txt b/storage/src/vespa/storage/tools/CMakeLists.txt index 3a00573cdd8..4bdcc7272db 100644 --- a/storage/src/vespa/storage/tools/CMakeLists.txt +++ b/storage/src/vespa/storage/tools/CMakeLists.txt @@ -15,14 +15,6 @@ vespa_add_executable(storage_generatedistributionbits_app AFTER storage_storageconfig ) -vespa_add_executable(storage_analyzedistribution_app - SOURCES - analyzedistribution.cpp - DEPENDS - storage - AFTER - storage_storageconfig -) vespa_add_executable(storage_storage-cmd_app SOURCES storage-cmd.cpp @@ -31,10 +23,3 @@ vespa_add_executable(storage_storage-cmd_app AFTER storage_storageconfig ) -vespa_add_executable(storage_statfs_app - SOURCES - statfs.cpp - DEPENDS - AFTER - storage_storageconfig -) diff --git a/storage/src/vespa/storage/tools/lib/.gitignore b/storage/src/vespa/storage/tools/lib/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/storage/src/vespa/storage/tools/lib/.gitignore +++ /dev/null diff --git a/storage/src/vespa/storage/tools/statfs.cpp b/storage/src/vespa/storage/tools/statfs.cpp deleted file mode 100644 index cc90b77e91d..00000000000 --- a/storage/src/vespa/storage/tools/statfs.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <iostream> -#include <sys/vfs.h> -#include <vespa/vespalib/util/programoptions.h> -#include <vespa/vespalib/io/fileutil.h> - -struct Options : public vespalib::ProgramOptions { - bool showSyntaxPage; - std::string _filename; - - Options(int argc, const char* const* argv); - ~Options(); -}; - -Options::Options(int argc, const char* const* argv) - : vespalib::ProgramOptions(argc, argv), - showSyntaxPage(false) -{ - setSyntaxMessage( - "Utility program for checking output of statfs." - ); - addOption("h help", showSyntaxPage, false, - "Shows this help page"); - addArgument("file", _filename, "File to use when calling statfs()"); -} -Options::~Options() {} - - -int main(int argc, char** argv) { - Options o(argc, argv); - o.parse(); - - if (o.showSyntaxPage) { - o.writeSyntaxPage(std::cerr); - exit(1); - } - - if (!vespalib::fileExists(o._filename)) { - std::cerr << "Cannot use statfs on non-existing file '" << o._filename - << "'.\n"; - exit(1); - } - - struct statfs buf; - if (statfs(o._filename.c_str(), &buf) == 0) { - std::cerr << "f_type " << buf.f_type << "\n" - << "f_bsize " << buf.f_bsize << "\n" - << "f_blocks " << buf.f_blocks << "\n" - << "f_bfree " << buf.f_bfree << "\n" - << "f_bavail " << buf.f_bavail << "\n" - << "f_files " << buf.f_files << "\n" - << "f_ffree " << buf.f_ffree << "\n" - << "f_namelen " << buf.f_namelen << "\n"; - - uint64_t available = buf.f_bavail; - uint64_t total = buf.f_blocks; - available *= buf.f_bsize; - total *= buf.f_bsize; - - std::cerr << "\nAvailable " << available << " of total " << total - << "\n" << (100.0 * (total - available) / (double) total) - << " % full\n"; - } else { - std::cerr << "statfs() failed: " << errno << "\n"; - } -} diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp index 0e871720ad0..93030f699cc 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp @@ -8,21 +8,13 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <sstream> #include <cassert> +#include <atomic> namespace storage::api { namespace { -/** - * TODO - * From @vekterli - * I have no idea why the _lastMsgId update code masks away the 8 MSB, but if we assume it's probably for no - * overwhelmingly good reason we could replace this mutex with just a std::atomic<uint64_t> and do a relaxed - * fetch_add (shouldn't be any need for any barriers; ID increments have no other memory dependencies). U64 overflows - * here come under the category "never gonna happen in the real world". - * @balder agree - @vekterli fix in separate pull request :) - */ -vespalib::Lock _G_msgIdLock; +std::atomic<uint64_t> _G_lastMsgId(1000); } @@ -122,8 +114,8 @@ MessageType::MessageType::get(Id id) return *it->second; } MessageType::MessageType(vespalib::stringref name, Id id, - const MessageType* replyOf) - : _name(name), _id(id), _reply(NULL), _replyOf(replyOf) + const MessageType* replyOf) + : _name(name), _id(id), _reply(NULL), _replyOf(replyOf) { _codes[id] = this; if (_replyOf != 0) { @@ -134,7 +126,7 @@ MessageType::MessageType(vespalib::stringref name, Id id, } } -MessageType::~MessageType() {} +MessageType::~MessageType() = default; void MessageType::print(std::ostream& out, bool verbose, const std::string& indent) const @@ -255,15 +247,10 @@ StorageMessageAddress::print(vespalib::asciistream & out) const TransportContext::~TransportContext() = default; -StorageMessage::Id StorageMessage::_lastMsgId = 1000; - StorageMessage::Id StorageMessage::generateMsgId() { - vespalib::LockGuard sync(_G_msgIdLock); - Id msgId = _lastMsgId++; - _lastMsgId &= ((Id(-1) << 8) >> 8); - return msgId; + return _G_lastMsgId.fetch_add(1, std::memory_order_relaxed); } StorageMessage::StorageMessage(const MessageType& type, Id id) @@ -290,9 +277,7 @@ StorageMessage::~StorageMessage() = default; void StorageMessage::setNewMsgId() { - vespalib::LockGuard sync(_G_msgIdLock); - _msgId = _lastMsgId++; - _lastMsgId &= ((Id(-1) << 8) >> 8); + _msgId = generateMsgId(); } vespalib::string diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h index ffbc24cd724..415bd7717f2 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h +++ b/storageapi/src/vespa/storageapi/messageapi/storagemessage.h @@ -349,11 +349,6 @@ public: static const char* getPriorityString(Priority); private: - static Id _lastMsgId; - - StorageMessage& operator=(const StorageMessage&); - StorageMessage(const StorageMessage&); - mutable std::unique_ptr<TransportContext> _transportContext; protected: @@ -372,6 +367,8 @@ protected: static document::Bucket getDummyBucket() { return document::Bucket(document::BucketSpace::invalid(), document::BucketId()); } public: + StorageMessage& operator=(const StorageMessage&) = delete; + StorageMessage(const StorageMessage&) = delete; virtual ~StorageMessage(); Id getMsgId() const { return _msgId; } diff --git a/travis/travis-build-full.sh b/travis/travis-build-full.sh index f44e5e55d7a..2f565e43605 100755 --- a/travis/travis-build-full.sh +++ b/travis/travis-build-full.sh @@ -16,7 +16,7 @@ ccache -p cd ${SOURCE_DIR} env VESPA_MAVEN_EXTRA_OPTS="--no-snapshot-updates --batch-mode --threads ${NUM_THREADS}" sh ./bootstrap.sh java mvn -V install --no-snapshot-updates --batch-mode --threads ${NUM_THREADS} -bash ${SOURCE_DIR}/bootstrap-cmake.sh ${SOURCE_DIR} +cmake3 -DVESPA_UNPRIVILEGED=no . make -j ${NUM_THREADS} ctest3 --output-on-failure -j ${NUM_THREADS} ccache --show-stats diff --git a/vespalib/src/apps/vespa-drop-file-from-cache/drop_file_from_cache.cpp b/vespalib/src/apps/vespa-drop-file-from-cache/drop_file_from_cache.cpp index 7e15885270c..280edf729b6 100644 --- a/vespalib/src/apps/vespa-drop-file-from-cache/drop_file_from_cache.cpp +++ b/vespalib/src/apps/vespa-drop-file-from-cache/drop_file_from_cache.cpp @@ -21,12 +21,14 @@ int main(int argc, char **argv) { } int retval = 0; +#ifdef __linux__ int err = posix_fadvise(fh, 0, 0, POSIX_FADV_DONTNEED); if (err != 0) { std::error_code ec(errno, std::system_category()); fprintf(stderr, "posix_fadvise failed: %s\n", ec.message().c_str()); retval = 3; } +#endif close(fh); return retval; } diff --git a/vespalib/src/tests/io/fileutil/fileutiltest.cpp b/vespalib/src/tests/io/fileutil/fileutiltest.cpp index 8345a8dcb99..b0a08f4e7fd 100644 --- a/vespalib/src/tests/io/fileutil/fileutiltest.cpp +++ b/vespalib/src/tests/io/fileutil/fileutiltest.cpp @@ -481,61 +481,6 @@ TEST("require that copy constructor and assignment for vespalib::File works") } } -TEST("require that vespalib::LazyFile works") -{ - // Copy constructor - { - LazyFile file("myfile", File::CREATE, true); - LazyFile file2(file); - EXPECT_EQUAL(file.getFlags(), file2.getFlags()); - EXPECT_EQUAL(file.autoCreateDirectories(), file2.autoCreateDirectories()); - } - // Assignment - { - LazyFile file("myfile", File::CREATE, true); - LazyFile file2("targetfile", File::READONLY); - file = file2; - EXPECT_EQUAL(file.getFlags(), file2.getFlags()); - EXPECT_EQUAL(file.autoCreateDirectories(), file2.autoCreateDirectories()); - } - // Lazily write - { - LazyFile file("myfile", File::CREATE, true); - file.write("foo", 3, 0); - } - // Lazy stat - { - LazyFile file("myfile", File::CREATE, true); - EXPECT_EQUAL(3, file.getFileSize()); - file.close(); - - LazyFile file2("myfile", File::CREATE, true); - FileInfo info = file2.stat(); - EXPECT_EQUAL(3, info._size); - EXPECT_EQUAL(true, info._plainfile); - } - - // Lazy read - { - LazyFile file("myfile", File::CREATE, true); - std::vector<char> buf(10, ' '); - EXPECT_EQUAL(3u, file.read(&buf[0], 10, 0)); - EXPECT_EQUAL(std::string("foo"), std::string(&buf[0], 3)); - } - // Lazy resize - { - LazyFile file("myfile", File::CREATE, true); - file.resize(5); - EXPECT_EQUAL(5, file.getFileSize()); - } - // Lazy get file descriptor - { - LazyFile file("myfile", File::CREATE, true); - int fd = file.getFileDescriptor(); - ASSERT_TRUE(fd != -1); - } -} - TEST("require that vespalib::symlink works") { // Target exists diff --git a/vespalib/src/tests/slaveproc/slaveproc_test.cpp b/vespalib/src/tests/slaveproc/slaveproc_test.cpp index 7a39d4f1c7b..547da991211 100644 --- a/vespalib/src/tests/slaveproc/slaveproc_test.cpp +++ b/vespalib/src/tests/slaveproc/slaveproc_test.cpp @@ -18,7 +18,7 @@ TEST("simple run, ignore output, timeout") { TEST("simple run") { std::string out; - EXPECT_TRUE(SlaveProc::run("echo -n foo", out)); + EXPECT_TRUE(SlaveProc::run("/bin/echo -n foo", out)); EXPECT_EQUAL(out, "foo"); } diff --git a/vespalib/src/vespa/vespalib/io/fileutil.h b/vespalib/src/vespa/vespalib/io/fileutil.h index 1507513dbbc..59994819600 100644 --- a/vespalib/src/vespa/vespalib/io/fileutil.h +++ b/vespalib/src/vespa/vespalib/io/fileutil.h @@ -209,65 +209,6 @@ public: }; /** - * @brief A File instance that automatically opens once needed. - */ -class LazyFile : public File { - int _flags; - bool _autoCreateDirectories; - -public: - typedef std::unique_ptr<LazyFile> UP; - - LazyFile(vespalib::stringref filename, int flags, - bool autoCreateDirs = false) - : File(filename), - _flags(flags), - _autoCreateDirectories(autoCreateDirs) {} - - LazyFile(LazyFile& other) - : File(other), - _flags(other._flags), - _autoCreateDirectories(other._autoCreateDirectories) {} - - LazyFile& operator=(LazyFile& other) { - File::operator=(other); - _flags = other._flags; - _autoCreateDirectories = other._autoCreateDirectories; - return *this; - } - - int getFlags() const { return _flags; } - void setFlags(int flags) { _flags = flags; } - void setAutoCreateDirectories(bool autoCreate) - { _autoCreateDirectories = autoCreate; } - bool autoCreateDirectories() const { return _autoCreateDirectories; } - - int getFileDescriptor() const override { - if (!isOpen()) { - const_cast<LazyFile&>(*this).open(_flags, _autoCreateDirectories); - } - return File::getFileDescriptor(); - } - - void resize(off_t size) override { - if (!isOpen()) { open(_flags, _autoCreateDirectories); } - File::resize(size); - } - - off_t write(const void *buf, size_t bufsize, off_t offset) override { - if (!isOpen()) { open(_flags, _autoCreateDirectories); } - return File::write(buf, bufsize, offset); - } - - size_t read(void *buf, size_t bufsize, off_t offset) const override { - if (!isOpen()) { - const_cast<LazyFile&>(*this).open(_flags, _autoCreateDirectories); - } - return File::read(buf, bufsize, offset); - } -}; - -/** * Get the current working directory. * * @throw IoException On failure. diff --git a/vespalib/src/vespa/vespalib/util/time.cpp b/vespalib/src/vespa/vespalib/util/time.cpp index 46cf4806dfc..61a295c6dc3 100644 --- a/vespalib/src/vespa/vespalib/util/time.cpp +++ b/vespalib/src/vespa/vespalib/util/time.cpp @@ -52,3 +52,34 @@ Timer::waitAtLeast(duration dur, bool busyWait) { } } + +namespace std::chrono { + +/* + * This is a hack to avoid the slow clock computations on RHEL7/CentOS 7 due to using systemcalls. + * This brings cost down from 550-560ns to 18-19ns on a Intel Haswell 2680 cpu. + * We are providing the symbols here so they will take precedence over the ones in the standard library. + * We rely on the linker do handle correct symbol resolution. + * TODO: Once we are off the ancient platforms like Centos 7/ Rhel 7 we can drop this workaround. +*/ + +inline namespace _V2 { + +system_clock::time_point +system_clock::now() noexcept { + timespec tp; + clock_gettime(CLOCK_REALTIME, &tp); + return time_point(duration(chrono::seconds(tp.tv_sec) + + chrono::nanoseconds(tp.tv_nsec))); +} + +steady_clock::time_point +steady_clock::now() noexcept { + timespec tp; + clock_gettime(CLOCK_MONOTONIC, &tp); + return time_point(duration(chrono::seconds(tp.tv_sec) + + chrono::nanoseconds(tp.tv_nsec))); +} + +} +} |