diff options
| author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-10-17 09:16:06 +0200 |
|---|---|---|
| committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-10-17 09:16:06 +0200 |
| commit | 0045f14d25a026c298af4c27a7db96494637522e (patch) | |
| tree | 50d87e8cfb34995590de28289110c1c395a88ff2 | |
| parent | 8ae188767b3bd760ab366dbd48dcaf9e9d5359e0 (diff) | |
| parent | eeb81f7263c2e80c0de1dfb0198ecf2ea8d38512 (diff) | |
Merge with master
277 files changed, 1609 insertions, 1750 deletions
diff --git a/build_settings.cmake b/build_settings.cmake index 020056ce354..17ee05bff1b 100644 --- a/build_settings.cmake +++ b/build_settings.cmake @@ -77,3 +77,7 @@ if(EXTRA_LINK_DIRECTORY) endif() set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-rpath,${CMAKE_BUILD_RPATH}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-rpath,${CMAKE_BUILD_RPATH}") + +# Don't allow unresolved symbols in executables or shared libraries +set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined") +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-undefined") diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java index 6a5abee7cb3..ed254bf3e08 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java @@ -2,7 +2,6 @@ package com.yahoo.config.model.provision; import com.google.common.collect.ImmutableMap; -import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.net.HostName; import com.yahoo.text.XML; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java index dbfad0a1370..5b88361c15b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java @@ -141,7 +141,10 @@ public class ConstantTensorJsonValidator { * additionally, those for indexed bounded dimensions needs to fall within the dimension size. */ private void validateTensorCoordinate(TensorType.Dimension dimension) throws IOException { - assertNextTokenIs(JsonToken.VALUE_STRING); + final JsonToken token = parser.nextToken(); + if (token != JsonToken.VALUE_STRING) { + throw new InvalidConstantTensor(parser, String.format("Tensor coordinate is not a string (%s)", token.toString())); + } if (dimension instanceof TensorType.IndexedBoundDimension) { validateBoundedCoordinate((TensorType.IndexedBoundDimension) dimension); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java new file mode 100644 index 00000000000..7cc37938658 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java @@ -0,0 +1,70 @@ +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.path.Path; +import com.yahoo.searchdefinition.RankingConstant; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensor; +import com.yahoo.vespa.model.search.SearchDefinition; + +import java.io.FileNotFoundException; +import java.io.Reader; + +/** + * RankingConstantsValidator validates all constant tensors (ranking constants) bundled with an application package + * + * @author Vegard Sjonfjell + */ + +public class RankingConstantsValidator extends Validator { + private static class ExceptionMessageCollector { + public String combinedMessage; + public boolean exceptionsOccurred = false; + + public ExceptionMessageCollector(String messagePrelude) { + this.combinedMessage = messagePrelude; + } + + public ExceptionMessageCollector add(Throwable throwable, String rcName, String rcFilename) { + exceptionsOccurred = true; + combinedMessage += String.format("\nRanking constant \"%s\" (%s): %s", rcName, rcFilename, throwable.getMessage()); + return this; + } + } + + public static class TensorValidationFailed extends RuntimeException { + public TensorValidationFailed(String message) { + super(message); + } + } + + @Override + public void validate(VespaModel model, DeployState deployState) { + final ApplicationPackage applicationPackage = deployState.getApplicationPackage(); + final ExceptionMessageCollector exceptionMessageCollector = new ExceptionMessageCollector("Failed to validate constant tensor file(s):"); + + for (SearchDefinition sd : deployState.getSearchDefinitions()) { + for (RankingConstant rc : sd.getSearch().getRankingConstants()) { + try { + validateRankingConstant(rc, applicationPackage); + } catch (InvalidConstantTensor | FileNotFoundException ex) { + exceptionMessageCollector.add(ex, rc.getName(), rc.getFileName()); + } + } + } + + if (exceptionMessageCollector.exceptionsOccurred) { + throw new TensorValidationFailed(exceptionMessageCollector.combinedMessage); + } + } + + public static void validateRankingConstant(RankingConstant rankingConstant, ApplicationPackage applicationPackage) throws FileNotFoundException { + final ApplicationFile tensorApplicationFile = applicationPackage.getFile(Path.fromString(rankingConstant.getFileName())); + final Reader tensorReader = tensorApplicationFile.createReader(); + + ConstantTensorJsonValidator tensorValidator = new ConstantTensorJsonValidator(tensorReader, rankingConstant.getTensorType()); + tensorValidator.validate(); + } +}
\ No newline at end of file diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 819d2638614..68873ebf500 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -51,6 +51,7 @@ public class Validation { new RankSetupValidator(force).validate(model, deployState); new NoPrefixForIndexes().validate(model, deployState); new DeploymentFileValidator().validate(model, deployState); + new RankingConstantsValidator().validate(model, deployState); Optional<Model> currentActiveModel = deployState.getPreviousModel(); if (currentActiveModel.isPresent() && (currentActiveModel.get() instanceof VespaModel)) diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd new file mode 100644 index 00000000000..4b0c4e297a6 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/searchdefinitions/simple.sd @@ -0,0 +1,29 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search simple { + document simple {} + + constant constant_tensor_1 { + file: tensors/constant_tensor_1.json + type: tensor(x[], y[]) + } + + constant constant_tensor_2 { + file: tensors/constant_tensor_2.json + type: tensor(x[]) + } + + constant constant_tensor_3 { + file: tensors/constant_tensor_3.json + type: tensor(cpp{}, d{}) + } + + constant constant_tensor_4 { + file: tensors/constant_tensor_4.json + type: tensor(x{}, y{}) + } + + constant constant_tensor_5 { + file: tensors/constant_tensor_5.json + type: tensor(x[], y[], z[]) + } +} diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/services.xml b/config-model/src/test/cfg/application/validation/ranking_constants_fail/services.xml new file mode 100644 index 00000000000..66af18ed7bc --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/services.xml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services> + <admin version="2.0"> + <adminserver hostalias="node1" /> + <logserver hostalias="node1" /> + </admin> + <content version="1.0"> + <redundancy>1</redundancy> + <documents> + <document type='simple' mode="index"/> + </documents> + <nodes> + <node hostalias='node1' distribution-key='0'/> + </nodes> + </content> +</services>
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_1.json b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_1.json new file mode 100644 index 00000000000..5b4fd45c0a3 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_1.json @@ -0,0 +1,8 @@ +{ + "cells": [ + { + "address": { "x": "3", "y": "2" }, + "value": 2.0 + } + ] +} diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_2.json b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_2.json new file mode 100644 index 00000000000..d54daa0bd89 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_2.json @@ -0,0 +1,8 @@ +{ + "cells": [ + { + "address": { "x": 5 }, + "value": 42.0 + } + ] +} diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_3.json b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_3.json new file mode 100644 index 00000000000..e44d0ca0569 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_3.json @@ -0,0 +1,12 @@ +{ + "cells": [ + { + "address": { "cpp": "bjarne stroustrup", "d": "andrei alexandrescu" }, + "value": 99.5 + }, + { + "address": { "cpp": "stephan lavavej", "cd": "walter bright" }, + "value": 47.0 + } + ] +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_4.json b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_4.json new file mode 100644 index 00000000000..46439be5e3f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_4.json @@ -0,0 +1,8 @@ +{ + "cells": [ + { + "address": { "x": "3", "z": "-99" }, + "value": 23 + } + ] +} diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_5.json b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_5.json new file mode 100644 index 00000000000..01d9204084f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_fail/tensors/constant_tensor_5.json @@ -0,0 +1,8 @@ +{ + "cells": [ + { + "address": { "x": "32", "y": "9", "z": "45" }, + "value": 3 + } + ] +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd b/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd new file mode 100644 index 00000000000..4b0c4e297a6 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/searchdefinitions/simple.sd @@ -0,0 +1,29 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +search simple { + document simple {} + + constant constant_tensor_1 { + file: tensors/constant_tensor_1.json + type: tensor(x[], y[]) + } + + constant constant_tensor_2 { + file: tensors/constant_tensor_2.json + type: tensor(x[]) + } + + constant constant_tensor_3 { + file: tensors/constant_tensor_3.json + type: tensor(cpp{}, d{}) + } + + constant constant_tensor_4 { + file: tensors/constant_tensor_4.json + type: tensor(x{}, y{}) + } + + constant constant_tensor_5 { + file: tensors/constant_tensor_5.json + type: tensor(x[], y[], z[]) + } +} diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/services.xml b/config-model/src/test/cfg/application/validation/ranking_constants_ok/services.xml new file mode 100644 index 00000000000..be6672f415f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/services.xml @@ -0,0 +1,17 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services> + <admin version="2.0"> + <adminserver hostalias="node1" /> + <logserver hostalias="node1" /> + </admin> + <content version="1.0"> + <redundancy>1</redundancy> + <documents> + <document type='simple' mode="index"/> + </documents> + <nodes> + <node hostalias='node1' distribution-key='0'/> + </nodes> + </content> +</services> diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_1.json b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_1.json new file mode 100644 index 00000000000..5b4fd45c0a3 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_1.json @@ -0,0 +1,8 @@ +{ + "cells": [ + { + "address": { "x": "3", "y": "2" }, + "value": 2.0 + } + ] +} diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_2.json b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_2.json new file mode 100644 index 00000000000..af3a02745f3 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_2.json @@ -0,0 +1,8 @@ +{ + "cells": [ + { + "address": { "x": "5" }, + "value": 42.0 + } + ] +} diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_3.json b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_3.json new file mode 100644 index 00000000000..d06fcbee42a --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_3.json @@ -0,0 +1,12 @@ +{ + "cells": [ + { + "address": { "cpp": "bjarne stroustrup", "d": "andrei alexandrescu" }, + "value": 99.5 + }, + { + "address": { "cpp": "stephan lavavej", "d": "walter bright" }, + "value": 47.0 + } + ] +} diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_4.json b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_4.json new file mode 100644 index 00000000000..cb2273f7b11 --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_4.json @@ -0,0 +1,8 @@ +{ + "cells": [ + { + "address": { "x": "3", "y": "-99" }, + "value": 23 + } + ] +} diff --git a/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_5.json b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_5.json new file mode 100644 index 00000000000..01d9204084f --- /dev/null +++ b/config-model/src/test/cfg/application/validation/ranking_constants_ok/tensors/constant_tensor_5.json @@ -0,0 +1,8 @@ +{ + "cells": [ + { + "address": { "x": "32", "y": "9", "z": "45" }, + "value": 3 + } + ] +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index a8554c54867..3f768088f4a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -8,7 +8,6 @@ import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployProperties; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.model.test.TestRoot; import com.yahoo.config.provision.ApplicationId; @@ -24,10 +23,7 @@ import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.StatisticsComponent; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; import org.junit.Test; -import org.xml.sax.SAXException; -import java.io.IOException; -import java.util.List; import java.util.Set; import static org.hamcrest.CoreMatchers.is; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java index 912af4f63c1..cdf4d88148e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidatorTest.java @@ -133,7 +133,24 @@ public class ConstantTensorJsonValidatorTest { " }", " ]", "}")); + } + + @Test + public void ensure_that_tensor_coordinates_are_strings() { + expectedException.expect(InvalidConstantTensor.class); + expectedException.expectMessage("Tensor coordinate is not a string (VALUE_NUMBER_INT)"); + validateTensorJson( + TensorType.fromSpec("tensor(x[])"), + inputJsonToReader( + "{", + " 'cells': [", + " {", + " 'address': { 'x': 47 },", + " 'value': 33.0", + " }", + " ]", + "}")); } @Test @@ -152,7 +169,6 @@ public class ConstantTensorJsonValidatorTest { " }", " ]", "}")); - } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java new file mode 100644 index 00000000000..c2aee71063d --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java @@ -0,0 +1,28 @@ +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static com.yahoo.vespa.model.application.validation.RankingConstantsValidator.TensorValidationFailed; + +public class RankingConstantsValidatorTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Test + public void ensure_that_valid_ranking_constants_do_not_fail() { + new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/ranking_constants_ok/").create(); + } + + @Test + public void ensure_that_failing_ranking_constants_fails() { + expectedException.expect(TensorValidationFailed.class); + expectedException.expectMessage("Ranking constant \"constant_tensor_2\" (tensors/constant_tensor_2.json): Tensor coordinate is not a string (VALUE_NUMBER_INT)"); + expectedException.expectMessage("Ranking constant \"constant_tensor_3\" (tensors/constant_tensor_3.json): Tensor dimension \"cd\" does not exist"); + expectedException.expectMessage("Ranking constant \"constant_tensor_4\" (tensors/constant_tensor_4.json): Tensor dimension \"z\" does not exist"); + + new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/ranking_constants_fail/").create(); + } +}
\ No newline at end of file diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java index c068af49b5f..3c465db5e5c 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ApplicationId.java @@ -75,6 +75,7 @@ public final class ApplicationId implements Comparable<ApplicationId> { .build(); + // TODO: Remove this abhorrent layer violation public boolean isHostedVespaRoutingApplication() { return HOSTED_VESPA_TENANT.equals(tenant) && (ROUTING_APPLICATION.equals(application) || diff --git a/config/src/vespa/config/frt/protocol.cpp b/config/src/vespa/config/frt/protocol.cpp index 5c7af53d2af..43eba571083 100644 --- a/config/src/vespa/config/frt/protocol.cpp +++ b/config/src/vespa/config/frt/protocol.cpp @@ -59,7 +59,7 @@ const Memory RESPONSE_COMPRESSION_INFO_UNCOMPRESSED_SIZE = "uncompressedSize"; DecompressedData decompress_lz4(const char * input, uint32_t inputLen, int uncompressedLength) { - Alloc memory( DefaultAlloc::create(uncompressedLength)); + Alloc memory( Alloc::alloc(uncompressedLength)); int sz = LZ4_decompress_safe(input, static_cast<char *>(memory.get()), inputLen, uncompressedLength); if (sz >= 0 && sz != uncompressedLength) { if (LOG_WOULD_LOG(debug)) { diff --git a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java index be0eb0bc2e9..c599c06833b 100644 --- a/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java +++ b/container-search/src/main/java/com/yahoo/fs4/QueryPacket.java @@ -205,7 +205,7 @@ public class QueryPacket extends Packet { static final int QFLAG_ALLOW_ERRORPACKET = 0x00000004; static final int QFLAG_ESTIMATE = 0x00000080; static final int QFLAG_DROP_SORTDATA = 0x00004000; - static final int QFLAG_REPORT_COVERAGE = 0x00008000; + static final int QFLAG_REPORT_COVERAGE = 0x00008000; // always on static final int QFLAG_NO_RESULTCACHE = 0x00010000; static final int QFLAG_DUMP_FEATURES = 0x00040000; diff --git a/container-search/src/main/java/com/yahoo/prelude/templates/DefaultTemplateSet.java b/container-search/src/main/java/com/yahoo/prelude/templates/DefaultTemplateSet.java index 1a7c5a738be..1578c702034 100644 --- a/container-search/src/main/java/com/yahoo/prelude/templates/DefaultTemplateSet.java +++ b/container-search/src/main/java/com/yahoo/prelude/templates/DefaultTemplateSet.java @@ -82,9 +82,7 @@ public class DefaultTemplateSet extends UserTemplate<XMLWriter> { context.setUtf8Output("utf-8".equalsIgnoreCase(getRequestedEncoding(result.getQuery()))); writer.xmlHeader(getRequestedEncoding(result.getQuery())); writer.openTag(RESULT).attribute(TOTAL_HIT_COUNT,String.valueOf(result.getTotalHitCount())); - if (result.getQuery().getPresentation().getReportCoverage()) { - renderCoverageAttributes(result.getCoverage(false), writer); - } + renderCoverageAttributes(result.getCoverage(false), writer); renderTime(writer, result); writer.closeStartTag(); } diff --git a/container-search/src/main/java/com/yahoo/search/Result.java b/container-search/src/main/java/com/yahoo/search/Result.java index b6a88200084..b96c0cf42eb 100644 --- a/container-search/src/main/java/com/yahoo/search/Result.java +++ b/container-search/src/main/java/com/yahoo/search/Result.java @@ -311,10 +311,6 @@ public final class Result extends com.yahoo.processing.Response implements Clone public void setCoverage(Coverage coverage) { this.coverage = coverage; } - // Coverage a part of tracing? - // Coverage logic might me moved around, but it should not be a part of tracing. - // Coverage is status information about access to a corpus, tracing is voluntary, - // diagnostic search status. /** * Returns coverage information * diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index 01e56e73cae..733e0aa43d9 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -287,8 +287,6 @@ public class SearchHandler extends LoggingRequestHandler { Query query = new Query(request, queryProfile); boolean benchmarkCoverage = VespaHeaders.benchmarkCoverage(benchmarkOutput, request.getJDiscRequest().headers()); - if (benchmarkCoverage) - query.getPresentation().setReportCoverage(true); // Find and execute search chain if we have a valid query String invalidReason = query.validate(); diff --git a/container-search/src/main/java/com/yahoo/search/query/Presentation.java b/container-search/src/main/java/com/yahoo/search/query/Presentation.java index 466ddf88299..cef3d46b947 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Presentation.java +++ b/container-search/src/main/java/com/yahoo/search/query/Presentation.java @@ -64,9 +64,6 @@ public class Presentation implements Cloneable { /** The summary class to be shown */ private String summary = null; - /** Whether coverage information (how much of the indices was searched should be included in the result */ - private boolean reportCoverage=false; - /** The name of the renderer to use for rendering the hits. */ private ComponentSpecification format = RendererRegistry.defaultRendererId.toSpecification(); @@ -99,11 +96,15 @@ public class Presentation implements Cloneable { /** Sets whether matching query terms should be bolded in the result */ public void setBolding(boolean bolding) { this.bolding = bolding; } - /** Returns whether coverage information should be returned in the result, if available. Default is false */ - public boolean getReportCoverage() { return reportCoverage; } + /** @deprecated coverage information is always returned */ + @Deprecated + // TODO: Remove on Vespa 7 + public boolean getReportCoverage() { return true; } - /** Sets whether coverage information should be returned in the result, if available */ - public void setReportCoverage(boolean reportCoverage) { this.reportCoverage=reportCoverage; } + /** @deprecated coverage information is always returned */ + @Deprecated + // TODO: Remove on Vespa 7 + public void setReportCoverage(boolean ignored) { } /** Get the name of the format desired for result rendering. */ @NonNull diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java index cd4e02dc768..7d2ebd2a0ec 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/QueryProperties.java @@ -120,7 +120,7 @@ public class QueryProperties extends Properties { else if (key.size()==2 && key.first().equals(Presentation.PRESENTATION)) { if (key.last().equals(Presentation.BOLDING)) return query.getPresentation().getBolding(); if (key.last().equals(Presentation.SUMMARY)) return query.getPresentation().getSummary(); - if (key.last().equals(Presentation.REPORT_COVERAGE)) return query.getPresentation().getReportCoverage(); + if (key.last().equals(Presentation.REPORT_COVERAGE)) return true; // TODO: Remove this line on Vespa 7 if (key.last().equals(Presentation.FORMAT)) return query.getPresentation().getFormat(); if (key.last().equals(Presentation.TIMING)) return query.getPresentation().getTiming(); if (key.last().equals(Presentation.SUMMARY_FIELDS)) return query.getPresentation().getSummaryFields(); @@ -227,15 +227,13 @@ public class QueryProperties extends Properties { query.getPresentation().setBolding(asBoolean(value, true)); else if (key.last().equals(Presentation.SUMMARY)) query.getPresentation().setSummary(asString(value, "")); - else if (key.last().equals(Presentation.REPORT_COVERAGE)) - query.getPresentation().setReportCoverage(asBoolean(value,true)); else if (key.last().equals(Presentation.FORMAT)) query.getPresentation().setFormat(asString(value,"")); else if (key.last().equals(Presentation.TIMING)) query.getPresentation().setTiming(asBoolean(value, true)); else if (key.last().equals(Presentation.SUMMARY_FIELDS)) query.getPresentation().setSummaryFields(asString(value,"")); - else + else if ( ! key.last().equals(Presentation.REPORT_COVERAGE)) // TODO: Change this line to "else" on Vespa 7 throwIllegalParameter(key.last(), Presentation.PRESENTATION); } else if (key.first().equals("rankfeature") || key.first().equals("featureoverride") ) { // featureoverride is deprecated diff --git a/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java index 3cf5d2cb3e5..ad1ad71992b 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/DefaultRenderer.java @@ -120,9 +120,7 @@ public final class DefaultRenderer extends AsynchronousSectionedRenderer<Result> utf8Output = "utf-8".equalsIgnoreCase(getRequestedEncoding(result.getQuery())); writer.xmlHeader(getRequestedEncoding(result.getQuery())); writer.openTag(RESULT).attribute(TOTAL_HIT_COUNT, String.valueOf(result.getTotalHitCount())); - if (result.getQuery().getPresentation().getReportCoverage()) { - renderCoverageAttributes(result.getCoverage(false), writer); - } + renderCoverageAttributes(result.getCoverage(false), writer); renderTime(writer, result); writer.closeStartTag(); } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java index d3039925013..e394a674a06 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/SyncDefaultRenderer.java @@ -136,9 +136,7 @@ public final class SyncDefaultRenderer extends Renderer { context.setUtf8Output("utf-8".equalsIgnoreCase(getRequestedEncoding(result.getQuery()))); writer.xmlHeader(getRequestedEncoding(result.getQuery())); writer.openTag(RESULT).attribute(TOTAL_HIT_COUNT,String.valueOf(result.getTotalHitCount())); - if (result.getQuery().getPresentation().getReportCoverage()) { - renderCoverageAttributes(result.getCoverage(false), writer); - } + renderCoverageAttributes(result.getCoverage(false), writer); renderTime(writer, result); writer.closeStartTag(); } diff --git a/container-search/src/test/java/com/yahoo/fs4/mplex/BackendTestCase.java b/container-search/src/test/java/com/yahoo/fs4/mplex/BackendTestCase.java index 52a66fa6db9..f9c14a81d13 100644 --- a/container-search/src/test/java/com/yahoo/fs4/mplex/BackendTestCase.java +++ b/container-search/src/test/java/com/yahoo/fs4/mplex/BackendTestCase.java @@ -1,9 +1,18 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.fs4.mplex; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import com.yahoo.container.search.Fs4Config; +import com.yahoo.fs4.BasicPacket; +import com.yahoo.fs4.ChannelTimeoutException; +import com.yahoo.fs4.PacketListener; +import com.yahoo.fs4.PingPacket; +import com.yahoo.fs4.QueryPacket; +import com.yahoo.fs4.mplex.Backend.BackendStatistics; +import com.yahoo.prelude.fastsearch.FS4ResourcePool; +import com.yahoo.search.Query; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; import java.io.IOException; import java.net.InetAddress; @@ -13,21 +22,9 @@ import java.net.Socket; import java.nio.ByteBuffer; import java.util.logging.Logger; -import com.yahoo.container.search.Fs4Config; -import com.yahoo.net.HostName; -import com.yahoo.net.LinuxInetAddress; -import com.yahoo.prelude.fastsearch.FS4ResourcePool; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import com.yahoo.fs4.BasicPacket; -import com.yahoo.fs4.ChannelTimeoutException; -import com.yahoo.fs4.PacketListener; -import com.yahoo.fs4.PingPacket; -import com.yahoo.fs4.QueryPacket; -import com.yahoo.fs4.mplex.Backend.BackendStatistics; -import com.yahoo.search.Query; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Test networking code for talking to dispatch. @@ -114,7 +111,6 @@ public class BackendTestCase { public MockServer() throws IOException { ServerSocket socket = new ServerSocket(0, 50, InetAddress.getLoopbackAddress()); - //ServerSocket socket = new ServerSocket(0, 50, LinuxInetAddress.getLocalHost()); host = (InetSocketAddress) socket.getLocalSocketAddress(); dispatch = new MockDispatch(socket); worker = new Thread(dispatch); diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java index 883c94cc5b7..609896a01bb 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/CreateContainerCommandImpl.java @@ -130,7 +130,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { List<String> labelList = labels.entrySet().stream() .map(entry -> entry.getKey() + "=" + entry.getValue()).collect(Collectors.toList()); - return "--name " + containerName + " " + return "--name " + containerName.asString() + " " + "--hostname " + hostName + " " + toRepeatedOption("--label", labelList) + toRepeatedOption("--env", environmentAssignments) @@ -139,7 +139,7 @@ class CreateContainerCommandImpl implements Docker.CreateContainerCommand { + toOptionalOption("--net", networkMode) + toOptionalOption("--ip", ipv4Address) + toOptionalOption("--ip6", ipv6Address) - + dockerImage; + + dockerImage.asString(); } private String generateRandomMACAddress() { diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java index 9142009a48f..5e5185bc6be 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/DockerImpl.java @@ -79,9 +79,6 @@ public class DockerImpl implements Docker { private CounterWrapper numberOfDockerDaemonFails; private final boolean hackAroundPullImageDueToJerseyConflicts; - private JerseyDockerCmdExecFactory dockerFactory; - private RemoteApiVersion remoteApiVersion; - // For testing DockerImpl(final DockerClient dockerClient) { hackAroundPullImageDueToJerseyConflicts = false; @@ -89,7 +86,7 @@ public class DockerImpl implements Docker { } // For testing - public DockerImpl(final DockerConfig config) { + DockerImpl(final DockerConfig config) { hackAroundPullImageDueToJerseyConflicts = false; // Fail fast initDockerConnection(config, 100 /* connect timeout millis */, false /* fallback to 1.23 on errors */); @@ -532,12 +529,13 @@ public class DockerImpl implements Docker { } } - private void initDockerConnection(final DockerConfig config, int connectTimeousMs, boolean fallbackTo123orErrors) { - dockerFactory = new JerseyDockerCmdExecFactory() + private void initDockerConnection(final DockerConfig config, int connectTimeoutMs, boolean fallbackTo123orErrors) { + JerseyDockerCmdExecFactory dockerFactory = new JerseyDockerCmdExecFactory() .withMaxPerRouteConnections(DOCKER_MAX_PER_ROUTE_CONNECTIONS) .withMaxTotalConnections(DOCKER_MAX_TOTAL_CONNECTIONS) - .withConnectTimeout(connectTimeousMs) + .withConnectTimeout(connectTimeoutMs) .withReadTimeout(DOCKER_READ_TIMEOUT_MILLIS); + RemoteApiVersion remoteApiVersion; try { remoteApiVersion = RemoteApiVersion.parseConfig(DockerClientImpl.getInstance( buildDockerClientConfig(config).build()) @@ -571,7 +569,7 @@ public class DockerImpl implements Docker { numberOfRunningContainersGauge = metricReceiver.declareGauge(dimensions, "containers.running"); numberOfDockerDaemonFails = metricReceiver.declareCounter(dimensions, "daemon.api_fails"); - // Some containers could already be running, count them and intialize to that value + // Some containers could already be running, count them and initialize to that value numberOfRunningContainersGauge.sample(getAllManagedContainers().size()); } } diff --git a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java index b943a2ac856..26cbaff720c 100644 --- a/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java +++ b/docker-api/src/main/java/com/yahoo/vespa/hosted/dockerapi/metrics/MetricReceiverWrapper.java @@ -66,7 +66,7 @@ public class MetricReceiverWrapper implements Iterable<MetricReceiverWrapper.Dim while (dimensionsIterator.hasNext()) { Dimensions dimension = dimensionsIterator.next(); if (dimension.dimensionsMap.containsKey("host") && dimension.dimensionsMap.get("host").equals(hostname)) { - metricsByDimensions.remove(dimension); + dimensionsIterator.remove(); } } } diff --git a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java index 95c71d0780e..a77776f53dc 100644 --- a/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java +++ b/docker-api/src/test/java/com/yahoo/vespa/hosted/dockerapi/DockerTest.java @@ -3,19 +3,12 @@ package com.yahoo.vespa.hosted.dockerapi; import com.github.dockerjava.api.model.Network; import com.github.dockerjava.core.command.BuildImageResultCallback; -import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper; -import org.junit.Ignore; import org.junit.Test; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.net.Inet6Address; import java.net.InetAddress; -import java.net.NetworkInterface; -import java.net.SocketException; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -35,33 +28,23 @@ import static org.junit.Assume.assumeTrue; * 1. Install Docker Toolbox, and start it (Docker Quick Start Terminal) (you can close terminal window afterwards) * 2. Run tests from IDE/mvn. * - * LINUX: - * 1. Remove Ignore annotations - * 2. Change ownership of docker.sock - * $ sudo chown <your username> /var/run/docker.sock - * 3. (Temporary) Manually create the docker network used by DockerImpl by running: - * $ sudo docker network create --ipv6 --gateway=<your local IPv6 address> --subnet=fe80::1/16 habla - * 4. (Temporary) Manually build docker test image. Inside src/test/resources/simple-ipv6-server run: - * $ sudo docker build -t "simple-ipv6-server:Dockerfile" . - * 5. (Temporary) Comment out createDockerImage() and shutdown() - * * @author valerijf * @author dybdahl */ public class DockerTest { private DockerImpl docker; - private static final boolean isMacOSX = System.getProperty("os.name").equals("Mac OS X"); + private static final OS operatingSystem = getSystemOS(); private static final String prefix = "/Users/" + System.getProperty("user.name") + "/.docker/machine/machines/default/"; private static final DockerConfig dockerConfig = new DockerConfig(new DockerConfig.Builder() - .caCertPath(isMacOSX ? prefix + "ca.pem" : "") - .clientCertPath(isMacOSX ? prefix + "cert.pem" : "") - .clientKeyPath(isMacOSX ? prefix + "key.pem" : "") - .uri(isMacOSX ? "tcp://192.168.99.100:2376" : "unix:///var/run/docker.sock")); + .caCertPath(operatingSystem == OS.Mac_OS_X ? prefix + "ca.pem" : "") + .clientCertPath(operatingSystem == OS.Mac_OS_X ? prefix + "cert.pem" : "") + .clientKeyPath(operatingSystem == OS.Mac_OS_X ? prefix + "key.pem" : "") + .uri(operatingSystem == OS.Mac_OS_X ? "tcp://192.168.99.100:2376" : "tcp://localhost:2376")); private static final DockerImage dockerImage = new DockerImage("simple-ipv6-server:Dockerfile"); @Test public void testGetAllManagedContainersNoContainersRunning() { - assumeTrue(isMacOSX); + assumeTrue(operatingSystem != OS.Unsupported); assumeTrue(dockerDaemonIsPresent()); List<Container> containers = docker.getAllManagedContainers(); @@ -94,6 +77,7 @@ public class DockerTest { public void testCreateDeleteImageCreateDeleteContainer() throws IOException, InterruptedException, ExecutionException { assumeTrue(dockerDaemonIsPresent()); createDockerImage(docker); + ContainerName containerName = new ContainerName("foo"); docker.stopContainer(containerName); docker.deleteContainer(containerName); @@ -117,25 +101,32 @@ public class DockerTest { assertFalse("Failed to delete " + dockerImage.asString() + " image", docker.imageIsDownloaded(dockerImage)); } - @Ignore @Test public void testDockerNetworking() throws InterruptedException, ExecutionException, IOException { + assumeTrue(dockerDaemonIsPresent()); + createDockerImage(docker); + String hostName1 = "docker10.test.yahoo.com"; String hostName2 = "docker11.test.yahoo.com"; ContainerName containerName1 = new ContainerName("test-container-1"); ContainerName containerName2 = new ContainerName("test-container-2"); - InetAddress inetAddress1 = Inet6Address.getByName("fe80::10"); - InetAddress inetAddress2 = Inet6Address.getByName("fe80::11"); - DockerImpl docker = new DockerImpl(dockerConfig, new MetricReceiverWrapper(MetricReceiver.nullImplementation)); + InetAddress inetAddress1 = InetAddress.getByName("172.18.0.10"); + InetAddress inetAddress2 = InetAddress.getByName("172.18.0.11"); + + docker.createContainerCommand(dockerImage, containerName1, hostName1) + .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).withIpAddress(inetAddress1).create(); + + docker.createContainerCommand(dockerImage, containerName2, hostName2) + .withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).withIpAddress(inetAddress2).create(); - docker.createContainerCommand(dockerImage, containerName1, hostName1).withIpAddress(inetAddress1).create(); - docker.createContainerCommand(dockerImage, containerName2, hostName2).withIpAddress(inetAddress2).create(); + docker.startContainer(containerName1); + docker.startContainer(containerName2); try { testReachabilityFromHost(containerName1, inetAddress1); testReachabilityFromHost(containerName2, inetAddress2); - String[] curlFromNodeToNode = new String[]{"curl", "-g", "http://[" + inetAddress2 + "%eth0]/ping"}; + String[] curlFromNodeToNode = new String[]{"curl", "-g", "http://" + inetAddress2 + "/ping"}; while (! docker.executeInContainer(containerName1, curlFromNodeToNode).isSuccess()) { Thread.sleep(20); } @@ -151,41 +142,40 @@ public class DockerTest { } } + private void createDockerTestNetworkIfNeeded() { + if (! docker.dockerClient.listNetworksCmd().withNameFilter(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).exec().isEmpty()) return; + + Network.Ipam ipam = new Network.Ipam().withConfig(new Network.Ipam.Config().withSubnet("172.18.0.0/16")); + docker.dockerClient.createNetworkCmd() + .withName(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME).withDriver("bridge").withIpam(ipam).exec(); + } + private boolean dockerDaemonIsPresent() { - if (!isMacOSX) { + if (docker != null) return true; + if (operatingSystem == OS.Unsupported) { System.out.println("This test does not support " + System.getProperty("os.name") + " yet, ignoring test."); return false; } + try { - docker = new DockerImpl(dockerConfig, new MetricReceiverWrapper(MetricReceiver.nullImplementation)); + docker = new DockerImpl(dockerConfig); + createDockerTestNetworkIfNeeded(); return true; } catch (Exception e) { System.out.println("Please install Docker Toolbox and start Docker Quick Start Terminal once, ignoring test."); + e.printStackTrace(); return false; } } private void testReachabilityFromHost(ContainerName containerName, InetAddress target) throws IOException, InterruptedException { - String[] curlNodeFromHost = {"curl", "-g", "http://[" + target.getHostAddress() + "%" + getInterfaceName() + "]/ping"}; + String[] curlNodeFromHost = {"curl", "-g", "http://" + target.getHostAddress() + "/ping"}; while (!exec(curlNodeFromHost).equals("pong\n")) { Thread.sleep(20); } assertTrue("Could not reach " + containerName.asString() + " from host", exec(curlNodeFromHost).equals("pong\n")); } - - /** - * Returns the display name of the bridge used by our custom docker network. This is used for routing in the - * network tests. The bridge is assumed to be the only IPv6 interface starting with "br-" - */ - private static String getInterfaceName() throws SocketException { - return Collections.list(NetworkInterface.getNetworkInterfaces()).stream() - .filter(networkInterface -> networkInterface.getDisplayName().startsWith("br-") && - networkInterface.getInterfaceAddresses().stream() - .anyMatch(ip -> ip.getAddress() instanceof Inet6Address)) - .findFirst().orElseThrow(RuntimeException::new).getDisplayName(); - } - /** * Synchronously executes a system process and returns its stdout. Based of {@link com.yahoo.system.ProcessExecuter} * but could not be reused because of import errors. @@ -208,38 +198,11 @@ public class DockerTest { return ret.toString(); } - /** - * Returns IPv6 address of on the "docker0" interface that can be reached by the containers - */ - private static String getLocalIPv6Address() throws SocketException { - return Collections.list(NetworkInterface.getNetworkInterfaces()).stream() - .filter(networkInterface -> networkInterface.getDisplayName().equals("docker0")) - .flatMap(i -> Collections.list(i.getInetAddresses()).stream()) - .filter(ip -> ip instanceof Inet6Address && ip.isLinkLocalAddress()) - .findFirst().orElseThrow(RuntimeException::new) - .getHostAddress().split("%")[0]; - } - - private void createSomeNetork() throws SocketException { - Network.Ipam ipam = new Network.Ipam().withConfig(new Network.Ipam.Config() - .withSubnet("fe80::1/16").withGateway(getLocalIPv6Address())); - // TODO: This needs to match the network name in DockerOperations!? - docker.dockerClient.createNetworkCmd().withDriver("bridge").withName("habla") - .withIpam(ipam).exec(); - } - - private void remoteNetwork() { - // Remove the network we created earlier - // TODO: This needs to match the network name in DockerOperations!? - docker.dockerClient.removeNetworkCmd("habla").exec(); - } - - // TODO: Do we need this? Rather use network created by DockerImpl, right? - void createDockerImage(DockerImpl docker) throws IOException, ExecutionException, InterruptedException { + private void createDockerImage(DockerImpl docker) throws IOException, ExecutionException, InterruptedException { try { docker.deleteImage(new DockerImage(dockerImage.asString())); } catch (Exception e) { - assertThat(e.getMessage(), is("Failed to delete docker image simple-ipv6-server:Dockerfile")); + assertThat(e.getMessage(), is("Failed to delete docker image " + dockerImage.asString())); } // Build the image locally @@ -248,4 +211,14 @@ public class DockerTest { .buildImageCmd(dockerFilePath) .withTag(dockerImage.asString()).exec(new BuildImageResultCallback()).awaitCompletion(); } + + private enum OS {Linux, Mac_OS_X, Unsupported} + + private static OS getSystemOS() { + switch (System.getProperty("os.name").toLowerCase()) { + case "linux": return OS.Linux; + case "mac os x": return OS.Mac_OS_X; + default: return OS.Unsupported; + } + } } diff --git a/docker-api/src/test/resources/simple-ipv6-server/src/server.py b/docker-api/src/test/resources/simple-ipv6-server/src/server.py index 320f311e0fc..c28c288bbca 100644 --- a/docker-api/src/test/resources/simple-ipv6-server/src/server.py +++ b/docker-api/src/test/resources/simple-ipv6-server/src/server.py @@ -28,14 +28,16 @@ class MyHandler(SimpleHTTPRequestHandler): return -class HTTPServerV6(HTTPServer): - address_family = socket.AF_INET6 +class DualHTTPServer(HTTPServer): + def __init__(self, address, handler): + self.address_family = socket.AF_INET6 if (':' in address[0]) else socket.AF_INET + HTTPServer.__init__(self, address, handler) -def main(): - server = HTTPServerV6(('::', 80), MyHandler) +def main(ipv6): + server = DualHTTPServer(('::' if ipv6 else '', 80), MyHandler) server.serve_forever() if __name__ == '__main__': - main() + main(False) diff --git a/document/src/vespa/document/fieldvalue/fieldvalue.cpp b/document/src/vespa/document/fieldvalue/fieldvalue.cpp index 8a9b332709b..20568ea2a5c 100644 --- a/document/src/vespa/document/fieldvalue/fieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/fieldvalue.cpp @@ -23,7 +23,6 @@ using vespalib::FieldBase; using vespalib::nbostream; -using vespalib::DefaultAlloc; namespace document { diff --git a/document/src/vespa/document/util/bytebuffer.cpp b/document/src/vespa/document/util/bytebuffer.cpp index 4c8116482d5..f6a0bfda3c9 100644 --- a/document/src/vespa/document/util/bytebuffer.cpp +++ b/document/src/vespa/document/util/bytebuffer.cpp @@ -22,7 +22,6 @@ #define LOG_DEBUG4(a,b,c,d) LOG_DEBUG1(vespalib::make_string(a,b,c,d)); using vespalib::alloc::Alloc; -using vespalib::DefaultAlloc; namespace document { @@ -60,7 +59,7 @@ ByteBuffer::ByteBuffer() : } ByteBuffer::ByteBuffer(size_t len) : - ByteBuffer(DefaultAlloc::create(len), len) + ByteBuffer(Alloc::alloc(len), len) { } @@ -115,7 +114,7 @@ ByteBuffer& ByteBuffer::operator=(const ByteBuffer & org) if (this != & org) { cleanUp(); if (org._len > 0 && org._buffer) { - DefaultAlloc::create(org._len + 1).swap(_ownedBuffer); + Alloc::alloc(org._len + 1).swap(_ownedBuffer); _buffer = static_cast<char *>(_ownedBuffer.get()); memcpy(_buffer,org._buffer,org._len); _buffer[org._len] = 0; @@ -190,7 +189,7 @@ ByteBuffer::sliceFrom(const ByteBuffer& buf, size_t from, size_t to) // throw (B // Slicing from someone that doesn't own their buffer, must make own copy. if (( buf._ownedBuffer.get() == NULL ) && (buf._bufHolder == NULL)) { cleanUp(); - DefaultAlloc::create(to-from + 1).swap(_ownedBuffer); + Alloc::alloc(to-from + 1).swap(_ownedBuffer); _buffer = static_cast<char *>(_ownedBuffer.get()); memcpy(_buffer, buf._buffer + from, to-from); _buffer[to-from] = 0; @@ -219,7 +218,7 @@ ByteBuffer::sliceFrom(const ByteBuffer& buf, size_t from, size_t to) // throw (B ByteBuffer* ByteBuffer::copyBuffer(const char* buffer, size_t len) { if (buffer && len) { - Alloc newBuf = DefaultAlloc::create(len + 1); + Alloc newBuf = Alloc::alloc(len + 1); memcpy(newBuf.get(), buffer, len); static_cast<char *>(newBuf.get())[len] = 0; return new ByteBuffer(std::move(newBuf), len); diff --git a/document/src/vespa/document/util/compressor.cpp b/document/src/vespa/document/util/compressor.cpp index 9f0c6913325..062722292c2 100644 --- a/document/src/vespa/document/util/compressor.cpp +++ b/document/src/vespa/document/util/compressor.cpp @@ -12,7 +12,6 @@ LOG_SETUP(".document.compressor"); #include <lz4hc.h> using vespalib::alloc::Alloc; -using vespalib::DefaultAlloc; using vespalib::ConstBufferRef; using vespalib::DataBuffer; using vespalib::make_string; @@ -30,10 +29,10 @@ LZ4Compressor::process(const CompressionConfig& config, const void * inputV, siz char * output(static_cast<char *>(outputV)); int sz(-1); if (config.compressionLevel > 6) { - Alloc state = DefaultAlloc::create(LZ4_sizeofStateHC()); + Alloc state = Alloc::alloc(LZ4_sizeofStateHC()); sz = LZ4_compressHC2_withStateHC(state.get(), input, output, inputLen, config.compressionLevel); } else { - Alloc state = DefaultAlloc::create(LZ4_sizeofState()); + Alloc state = Alloc::alloc(LZ4_sizeofState()); sz = LZ4_compress_withState(state.get(), input, output, inputLen); } if (sz != 0) { diff --git a/document/src/vespa/document/util/serializable.cpp b/document/src/vespa/document/util/serializable.cpp index 2e9ea321b8f..c6738374223 100644 --- a/document/src/vespa/document/util/serializable.cpp +++ b/document/src/vespa/document/util/serializable.cpp @@ -6,8 +6,6 @@ #include <stdio.h> #include <vespa/document/util/bytebuffer.h> -using vespalib::DefaultAlloc; - namespace document { IMPLEMENT_IDENTIFIABLE_ABSTRACT(Serializable, vespalib::Identifiable); diff --git a/fastlib/src/vespa/fastlib/io/bufferedfile.cpp b/fastlib/src/vespa/fastlib/io/bufferedfile.cpp index adf201a6e1e..8d5aef0c097 100644 --- a/fastlib/src/vespa/fastlib/io/bufferedfile.cpp +++ b/fastlib/src/vespa/fastlib/io/bufferedfile.cpp @@ -392,7 +392,7 @@ size_t computeBufLen(size_t buflen) Fast_BufferedFile::Fast_BufferedFile(FastOS_FileInterface *file, size_t bufferSize) : FastOS_FileInterface(), _fileleft(static_cast<uint64_t>(-1)), - _buf(vespalib::alloc::MMapAllocFactory::create(computeBufLen(bufferSize))), + _buf(vespalib::alloc::Alloc::allocMMap(computeBufLen(bufferSize))), _bufi(NULL), _bufe(NULL), _filepos(0), diff --git a/filedistribution/src/apps/filedistributor/CMakeLists.txt b/filedistribution/src/apps/filedistributor/CMakeLists.txt index 0a54560f112..1a7ef87bbac 100644 --- a/filedistribution/src/apps/filedistributor/CMakeLists.txt +++ b/filedistribution/src/apps/filedistributor/CMakeLists.txt @@ -10,7 +10,7 @@ vespa_add_executable(filedistribution_filedistributor_app filedistribution_filedistributorrpc filedistribution_common ) -target_compile_options(filedistribution_filedistributor_app PRIVATE -DTORRENT_DEBUG -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) +target_compile_options(filedistribution_filedistributor_app PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_add_target_system_dependency(filedistribution_filedistributor_app boost boost_system-mt-d) vespa_add_target_system_dependency(filedistribution_filedistributor_app boost boost_thread-mt-d) vespa_add_target_system_dependency(filedistribution_filedistributor_app boost boost_program_options-mt-d) diff --git a/filedistribution/src/apps/status/CMakeLists.txt b/filedistribution/src/apps/status/CMakeLists.txt index 10de2dbcbb8..f7503e442e3 100644 --- a/filedistribution/src/apps/status/CMakeLists.txt +++ b/filedistribution/src/apps/status/CMakeLists.txt @@ -7,7 +7,7 @@ vespa_add_executable(filedistribution_status-filedistribution_app filedistribution_filedistributionmodel filedistribution_common ) -target_compile_options(filedistribution_status-filedistribution_app PRIVATE -DTORRENT_DEBUG -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) +target_compile_options(filedistribution_status-filedistribution_app PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_add_target_system_dependency(filedistribution_status-filedistribution_app boost boost_system-mt-d) vespa_add_target_system_dependency(filedistribution_status-filedistribution_app boost boost_thread-mt-d) vespa_add_target_system_dependency(filedistribution_status-filedistribution_app boost boost_program_options-mt-d) diff --git a/filedistribution/src/tests/common/CMakeLists.txt b/filedistribution/src/tests/common/CMakeLists.txt index 10fae25892c..522732e92ab 100644 --- a/filedistribution/src/tests/common/CMakeLists.txt +++ b/filedistribution/src/tests/common/CMakeLists.txt @@ -4,7 +4,7 @@ vespa_add_executable(filedistribution_common_test_app TEST testCommon.cpp DEPENDS ) -target_compile_options(filedistribution_common_test_app PRIVATE -DTORRENT_DEBUG -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) +target_compile_options(filedistribution_common_test_app PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_add_target_system_dependency(filedistribution_common_test_app boost boost_system-mt-d) vespa_add_target_system_dependency(filedistribution_common_test_app boost boost_thread-mt-d) vespa_add_target_system_dependency(filedistribution_common_test_app boost boost_program_options-mt-d) diff --git a/filedistribution/src/tests/filedbmodelimpl/CMakeLists.txt b/filedistribution/src/tests/filedbmodelimpl/CMakeLists.txt index aded3e4f07e..f42761c5fdc 100644 --- a/filedistribution/src/tests/filedbmodelimpl/CMakeLists.txt +++ b/filedistribution/src/tests/filedbmodelimpl/CMakeLists.txt @@ -7,7 +7,7 @@ vespa_add_executable(filedistribution_filedbmodelimpl_test_app TEST filedistribution_common filedistribution_mocks ) -target_compile_options(filedistribution_filedbmodelimpl_test_app PRIVATE -DTORRENT_DEBUG -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) +target_compile_options(filedistribution_filedbmodelimpl_test_app PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_add_target_system_dependency(filedistribution_filedbmodelimpl_test_app boost boost_system-mt-d) vespa_add_target_system_dependency(filedistribution_filedbmodelimpl_test_app boost boost_thread-mt-d) vespa_add_target_system_dependency(filedistribution_filedbmodelimpl_test_app boost boost_program_options-mt-d) diff --git a/filedistribution/src/tests/filedownloader/CMakeLists.txt b/filedistribution/src/tests/filedownloader/CMakeLists.txt index 31c2f67738d..5c2b0210828 100644 --- a/filedistribution/src/tests/filedownloader/CMakeLists.txt +++ b/filedistribution/src/tests/filedownloader/CMakeLists.txt @@ -6,7 +6,7 @@ vespa_add_executable(filedistribution_filedownloader_test_app TEST filedistribution_filedistributionmodel filedistribution_common ) -target_compile_options(filedistribution_filedownloader_test_app PRIVATE -DTORRENT_DEBUG -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) +target_compile_options(filedistribution_filedownloader_test_app PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_add_target_system_dependency(filedistribution_filedownloader_test_app boost boost_system-mt-d) vespa_add_target_system_dependency(filedistribution_filedownloader_test_app boost boost_thread-mt-d) vespa_add_target_system_dependency(filedistribution_filedownloader_test_app boost boost_program_options-mt-d) diff --git a/filedistribution/src/tests/rpc/CMakeLists.txt b/filedistribution/src/tests/rpc/CMakeLists.txt index 88ed4527449..a631abec631 100644 --- a/filedistribution/src/tests/rpc/CMakeLists.txt +++ b/filedistribution/src/tests/rpc/CMakeLists.txt @@ -6,7 +6,7 @@ vespa_add_executable(filedistribution_rpc_test_app TEST filedistribution_filedistributorrpc filedistribution_common ) -target_compile_options(filedistribution_rpc_test_app PRIVATE -DTORRENT_DEBUG -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) +target_compile_options(filedistribution_rpc_test_app PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_add_target_system_dependency(filedistribution_rpc_test_app boost boost_system-mt-d) vespa_add_target_system_dependency(filedistribution_rpc_test_app boost boost_thread-mt-d) vespa_add_target_system_dependency(filedistribution_rpc_test_app boost boost_program_options-mt-d) diff --git a/filedistribution/src/tests/scheduler/CMakeLists.txt b/filedistribution/src/tests/scheduler/CMakeLists.txt index ffc569b480b..f23949477b0 100644 --- a/filedistribution/src/tests/scheduler/CMakeLists.txt +++ b/filedistribution/src/tests/scheduler/CMakeLists.txt @@ -7,7 +7,7 @@ vespa_add_executable(filedistribution_scheduler_test_app TEST filedistribution_filedistributionmodel filedistribution_common ) -target_compile_options(filedistribution_scheduler_test_app PRIVATE -DTORRENT_DEBUG -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) +target_compile_options(filedistribution_scheduler_test_app PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_add_target_system_dependency(filedistribution_scheduler_test_app boost boost_system-mt-d) vespa_add_target_system_dependency(filedistribution_scheduler_test_app boost boost_thread-mt-d) vespa_add_target_system_dependency(filedistribution_scheduler_test_app boost boost_program_options-mt-d) diff --git a/filedistribution/src/tests/status/CMakeLists.txt b/filedistribution/src/tests/status/CMakeLists.txt index 30037c518ac..e9e7401e87e 100644 --- a/filedistribution/src/tests/status/CMakeLists.txt +++ b/filedistribution/src/tests/status/CMakeLists.txt @@ -6,7 +6,7 @@ vespa_add_executable(filedistribution_status_test_app TEST filedistribution_filedistributionmodel filedistribution_common ) -target_compile_options(filedistribution_status_test_app PRIVATE -DTORRENT_DEBUG -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) +target_compile_options(filedistribution_status_test_app PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_add_target_system_dependency(filedistribution_status_test_app boost boost_system-mt-d) vespa_add_target_system_dependency(filedistribution_status_test_app boost boost_thread-mt-d) vespa_add_target_system_dependency(filedistribution_status_test_app boost boost_program_options-mt-d) diff --git a/filedistribution/src/tests/zkfacade/CMakeLists.txt b/filedistribution/src/tests/zkfacade/CMakeLists.txt index 2c8e05aca89..600abfd04c2 100644 --- a/filedistribution/src/tests/zkfacade/CMakeLists.txt +++ b/filedistribution/src/tests/zkfacade/CMakeLists.txt @@ -7,7 +7,7 @@ vespa_add_executable(filedistribution_zkfacade_test_app filedistribution_common filedistribution_mocks ) -target_compile_options(filedistribution_zkfacade_test_app PRIVATE -DTORRENT_DEBUG -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) +target_compile_options(filedistribution_zkfacade_test_app PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_add_target_system_dependency(filedistribution_zkfacade_test_app boost boost_system-mt-d) vespa_add_target_system_dependency(filedistribution_zkfacade_test_app boost boost_thread-mt-d) vespa_add_target_system_dependency(filedistribution_zkfacade_test_app boost boost_program_options-mt-d) diff --git a/filedistribution/src/tests/zkfiledbmodel/CMakeLists.txt b/filedistribution/src/tests/zkfiledbmodel/CMakeLists.txt index f1ee7ac3e84..3496c3a104a 100644 --- a/filedistribution/src/tests/zkfiledbmodel/CMakeLists.txt +++ b/filedistribution/src/tests/zkfiledbmodel/CMakeLists.txt @@ -7,7 +7,7 @@ vespa_add_executable(filedistribution_zkfiledbmodel_test_app TEST filedistribution_common filedistribution_mocks ) -target_compile_options(filedistribution_zkfiledbmodel_test_app PRIVATE -DTORRENT_DEBUG -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) +target_compile_options(filedistribution_zkfiledbmodel_test_app PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_add_target_system_dependency(filedistribution_zkfiledbmodel_test_app boost boost_system-mt-d) vespa_add_target_system_dependency(filedistribution_zkfiledbmodel_test_app boost boost_thread-mt-d) vespa_add_target_system_dependency(filedistribution_zkfiledbmodel_test_app boost boost_program_options-mt-d) diff --git a/filedistribution/src/vespa/filedistribution/common/CMakeLists.txt b/filedistribution/src/vespa/filedistribution/common/CMakeLists.txt index a82b2db4b75..1f54fe7c0d5 100644 --- a/filedistribution/src/vespa/filedistribution/common/CMakeLists.txt +++ b/filedistribution/src/vespa/filedistribution/common/CMakeLists.txt @@ -11,5 +11,5 @@ vespa_add_library(filedistribution_common STATIC vespa_logfwd.cpp DEPENDS ) -target_compile_options(filedistribution_common PRIVATE -DTORRENT_DEBUG -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) +target_compile_options(filedistribution_common PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_add_target_system_dependency(filedistribution_common boost boost_thread-mt-d) diff --git a/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt b/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt index 1e1baf01e61..89cf90a9b31 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt +++ b/filedistribution/src/vespa/filedistribution/distributor/CMakeLists.txt @@ -10,6 +10,6 @@ vespa_add_library(filedistribution_distributor STATIC state_server_impl.cpp DEPENDS ) -target_compile_options(filedistribution_distributor PRIVATE -DTORRENT_DEBUG -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) +target_compile_options(filedistribution_distributor PRIVATE -DTORRENT_DISABLE_ENCRYPTION -DTORRENT_DISABLE_DHT -DWITH_SHIPPED_GEOIP_H -DBOOST_ASIO_HASH_MAP_BUCKETS=1021 -DBOOST_EXCEPTION_DISABLE -DBOOST_ASIO_ENABLE_CANCELIO -DBOOST_ASIO_DYN_LINK -DTORRENT_LINKING_SHARED) vespa_generate_config(filedistribution_distributor filedistributor.def) install(FILES filedistributor.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp b/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp index d1b99482388..e042ec28870 100644 --- a/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp +++ b/filedistribution/src/vespa/filedistribution/distributor/filedownloader.cpp @@ -402,6 +402,9 @@ void FileDownloader::runEventLoop() { } catch (const vespalib::PortListenException & e) { LOG(warning, "Failed listening to torrent port : %s", e.what()); std::quick_exit(21); + } catch (const boost::filesystem::filesystem_error & e) { + LOG(warning, "Some boost file operations failed : %s", e.what()); + std::quick_exit(22); } } drain(); diff --git a/fnet/src/vespa/fnet/databuffer.cpp b/fnet/src/vespa/fnet/databuffer.cpp index 913196d6e72..578faba1e49 100644 --- a/fnet/src/vespa/fnet/databuffer.cpp +++ b/fnet/src/vespa/fnet/databuffer.cpp @@ -3,8 +3,6 @@ #include <vespa/fastos/fastos.h> #include <vespa/fnet/fnet.h> -using vespalib::DefaultAlloc; - FNET_DataBuffer::FNET_DataBuffer(uint32_t len) : _bufstart(NULL), _bufend(NULL), @@ -15,7 +13,7 @@ FNET_DataBuffer::FNET_DataBuffer(uint32_t len) len = 256; if (len > 0) { - DefaultAlloc::create(len).swap(_ownedBuf); + Alloc::alloc(len).swap(_ownedBuf); memset(_ownedBuf.get(), 0x55, len); _bufstart = static_cast<char *>(_ownedBuf.get()); assert(_bufstart != NULL); @@ -72,7 +70,7 @@ FNET_DataBuffer::Shrink(uint32_t newsize) return false; } - Alloc newBuf(DefaultAlloc::create(newsize)); + Alloc newBuf(Alloc::alloc(newsize)); memset(newBuf.get(), 0x55, newsize); memcpy(newBuf.get(), _datapt, GetDataLen()); _ownedBuf.swap(newBuf); @@ -97,7 +95,7 @@ FNET_DataBuffer::Pack(uint32_t needbytes) while (bufsize - GetDataLen() < needbytes) bufsize *= 2; - Alloc newBuf(DefaultAlloc::create(bufsize)); + Alloc newBuf(Alloc::alloc(bufsize)); memset(newBuf.get(), 0x55, bufsize); memcpy(newBuf.get(), _datapt, GetDataLen()); _ownedBuf.swap(newBuf); diff --git a/fnet/src/vespa/fnet/frt/values.h b/fnet/src/vespa/fnet/frt/values.h index 8ab8914d31f..7e889466941 100644 --- a/fnet/src/vespa/fnet/frt/values.h +++ b/fnet/src/vespa/fnet/frt/values.h @@ -71,13 +71,14 @@ class FRT_Values public: class LocalBlob : public FRT_ISharedBlob { + using Alloc = vespalib::alloc::Alloc; public: - LocalBlob(vespalib::alloc::Alloc data, uint32_t len) : + LocalBlob(Alloc data, uint32_t len) : _data(std::move(data)), _len(len) { } LocalBlob(const char *data, uint32_t len) : - _data(vespalib::DefaultAlloc::create(len)), + _data(Alloc::alloc(len)), _len(len) { if (data != NULL) { @@ -85,7 +86,7 @@ public: } } void addRef() override {} - void subRef() override { vespalib::alloc::Alloc().swap(_data); } + void subRef() override { Alloc().swap(_data); } uint32_t getLen() override { return _len; } const char *getData() override { return static_cast<const char *>(_data.get()); } char *getInternalData() { return static_cast<char *>(_data.get()); } @@ -93,7 +94,7 @@ public: LocalBlob(const LocalBlob &); LocalBlob &operator=(const LocalBlob &); - vespalib::alloc::Alloc _data; + Alloc _data; uint32_t _len; }; diff --git a/fsa/src/vespa/fsamanagers/CMakeLists.txt b/fsa/src/vespa/fsamanagers/CMakeLists.txt index e5de5bc1b1a..0cc044d2dc8 100644 --- a/fsa/src/vespa/fsamanagers/CMakeLists.txt +++ b/fsa/src/vespa/fsamanagers/CMakeLists.txt @@ -9,6 +9,7 @@ vespa_add_library(fsamanagers singleton.cpp INSTALL lib64 DEPENDS + fsa ) find_package(Threads REQUIRED) target_link_libraries(fsamanagers PUBLIC ${CMAKE_THREAD_LIBS_INIT}) diff --git a/functions.cmake b/functions.cmake index 7db3d2edaf1..4e679509735 100644 --- a/functions.cmake +++ b/functions.cmake @@ -431,6 +431,15 @@ function(vespa_workaround_gcc_bug_67055 SOURCE_FILE) endif() endfunction() +function(vespa_workaround_set_gcc_march_skylake_avx512_if_supported SOURCE_FILE) + if(CMAKE_COMPILER_IS_GNUCC) + execute_process(COMMAND ${CMAKE_CPP_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_VERSION) + if (GCC_VERSION VERSION_GREATER "5.2") + set_source_files_properties(${SOURCE_FILE} PROPERTIES COMPILE_FLAGS -march=skylake-avx512) + endif() + endif() +endfunction() + macro(__initialize_module) # Set a couple of useful variables for this module set(MODULE_ROOT ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/memfilepersistence/CMakeLists.txt b/memfilepersistence/CMakeLists.txt index 005a1373b75..0f890465287 100644 --- a/memfilepersistence/CMakeLists.txt +++ b/memfilepersistence/CMakeLists.txt @@ -31,6 +31,7 @@ vespa_define_module( src/tests/conformance src/tests/device src/tests/init + src/tests/helper src/tests/spi src/tests/tools ) diff --git a/memfilepersistence/src/tests/CMakeLists.txt b/memfilepersistence/src/tests/CMakeLists.txt index 24863d6e837..119577bd777 100644 --- a/memfilepersistence/src/tests/CMakeLists.txt +++ b/memfilepersistence/src/tests/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_executable(memfilepersistence_testrunner_app TEST SOURCES - testhelper.cpp testrunner.cpp DEPENDS memfilepersistence_testconformance @@ -9,6 +8,5 @@ vespa_add_executable(memfilepersistence_testrunner_app TEST memfilepersistence_testinit memfilepersistence_testspi memfilepersistence_testtools - memfilepersistence ) vespa_add_test(NAME memfilepersistence_testrunner_app COMMAND memfilepersistence_testrunner_app) diff --git a/memfilepersistence/src/tests/device/CMakeLists.txt b/memfilepersistence/src/tests/device/CMakeLists.txt index 845c70ae8e3..21f45aba6e8 100644 --- a/memfilepersistence/src/tests/device/CMakeLists.txt +++ b/memfilepersistence/src/tests/device/CMakeLists.txt @@ -7,4 +7,5 @@ vespa_add_library(memfilepersistence_testdevices devicemappertest.cpp partitionmonitortest.cpp DEPENDS + memfilepersistence ) diff --git a/memfilepersistence/src/tests/helper/CMakeLists.txt b/memfilepersistence/src/tests/helper/CMakeLists.txt new file mode 100644 index 00000000000..f7dd097e2b2 --- /dev/null +++ b/memfilepersistence/src/tests/helper/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(memfilepersistence_testhelper + SOURCES + testhelper.cpp + DEPENDS + vdstestlib +) diff --git a/memfilepersistence/src/tests/testhelper.cpp b/memfilepersistence/src/tests/helper/testhelper.cpp index 39bd1d0968b..13950026b22 100644 --- a/memfilepersistence/src/tests/testhelper.cpp +++ b/memfilepersistence/src/tests/helper/testhelper.cpp @@ -1,6 +1,6 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <tests/testhelper.h> +#include <tests/helper/testhelper.h> #include <vespa/log/log.h> #include <vespa/vespalib/io/fileutil.h> diff --git a/memfilepersistence/src/tests/testhelper.h b/memfilepersistence/src/tests/helper/testhelper.h index 4445086d300..4445086d300 100644 --- a/memfilepersistence/src/tests/testhelper.h +++ b/memfilepersistence/src/tests/helper/testhelper.h diff --git a/memfilepersistence/src/tests/init/CMakeLists.txt b/memfilepersistence/src/tests/init/CMakeLists.txt index ebc4738a8c4..db50e017c08 100644 --- a/memfilepersistence/src/tests/init/CMakeLists.txt +++ b/memfilepersistence/src/tests/init/CMakeLists.txt @@ -3,4 +3,5 @@ vespa_add_library(memfilepersistence_testinit SOURCES filescannertest.cpp DEPENDS + memfilepersistence ) diff --git a/memfilepersistence/src/tests/spi/CMakeLists.txt b/memfilepersistence/src/tests/spi/CMakeLists.txt index d5dade96f57..38dadb9fd26 100644 --- a/memfilepersistence/src/tests/spi/CMakeLists.txt +++ b/memfilepersistence/src/tests/spi/CMakeLists.txt @@ -17,4 +17,6 @@ vespa_add_library(memfilepersistence_testspi buffered_file_writer_test.cpp buffer_test.cpp DEPENDS + memfilepersistence_testhelper + memfilepersistence ) diff --git a/memfilepersistence/src/tests/spi/memfiletestutils.h b/memfilepersistence/src/tests/spi/memfiletestutils.h index a13b902a214..796294fcaa3 100644 --- a/memfilepersistence/src/tests/spi/memfiletestutils.h +++ b/memfilepersistence/src/tests/spi/memfiletestutils.h @@ -13,7 +13,7 @@ #pragma once #include <vespa/memfilepersistence/memfile/memfilecache.h> -#include <tests/testhelper.h> +#include <tests/helper/testhelper.h> #include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/memfilepersistence/spi/memfilepersistenceprovider.h> #include <vespa/document/base/testdocman.h> diff --git a/memfilepersistence/src/tests/tools/CMakeLists.txt b/memfilepersistence/src/tests/tools/CMakeLists.txt index aef718c7633..4a3b4fb3061 100644 --- a/memfilepersistence/src/tests/tools/CMakeLists.txt +++ b/memfilepersistence/src/tests/tools/CMakeLists.txt @@ -4,4 +4,6 @@ vespa_add_library(memfilepersistence_testtools dumpslotfiletest.cpp vdsdisktooltest.cpp DEPENDS + memfilepersistence_testspi + memfilepersistence ) diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/buffer.cpp b/memfilepersistence/src/vespa/memfilepersistence/mapper/buffer.cpp index a998bb7d90e..acf7ac6a69d 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/buffer.cpp +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/buffer.cpp @@ -5,7 +5,6 @@ #include <algorithm> #include <stdlib.h> -using vespalib::DefaultAlloc; using vespalib::alloc::MemoryAllocator; using vespalib::alloc::Alloc; @@ -16,7 +15,7 @@ namespace memfile { // It is crucial that any backing buffer type returns an address that is // 512-byte aligned, or direct IO will scream at us and fail everything. Buffer::Buffer(size_t size) - : _buffer(DefaultAlloc::create(size, MemoryAllocator::HUGEPAGE_SIZE, 512)), + : _buffer(Alloc::alloc(size, MemoryAllocator::HUGEPAGE_SIZE, 512)), _size(size) { } diff --git a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.h b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.h index c564893a154..d305d6a8e25 100644 --- a/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.h +++ b/memfilepersistence/src/vespa/memfilepersistence/mapper/simplememfileiobuffer.h @@ -77,7 +77,7 @@ public: typedef vespalib::LinkedPtr<SharedBuffer> LP; explicit SharedBuffer(size_t totalSize) - : _buf(vespalib::alloc::MMapAllocFactory::create(totalSize)), + : _buf(vespalib::alloc::Alloc::allocMMap(totalSize)), _usedSize(0) { } diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/Identity.java b/messagebus/src/main/java/com/yahoo/messagebus/network/Identity.java index 45887b072ab..585fe680ba9 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/Identity.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/Identity.java @@ -1,12 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.messagebus.network; -import com.yahoo.log.LogLevel; import com.yahoo.net.HostName; -import com.yahoo.net.LinuxInetAddress; - -import java.net.Inet6Address; -import java.net.InetAddress; /** * This class encapsulates the identity of the application that uses this instance of message bus. This identity @@ -28,11 +23,7 @@ public class Identity { * @param configId The config identifier for the application. */ public Identity(String configId) { - InetAddress addr = LinuxInetAddress.getLocalHost(); // try hard to get a resolvable address - if (addr instanceof Inet6Address) // - hostname = HostName.getLocalhost(); // ... but fallback to hostname if we get an IPv6 address - else - hostname = addr.getCanonicalHostName(); + hostname = HostName.getLocalhost(); // ... but fallback to hostname if we get an IPv6 address servicePrefix = configId; } diff --git a/messagebus/src/vespa/messagebus/blob.h b/messagebus/src/vespa/messagebus/blob.h index 86e317d6e79..ab915e7ce1c 100644 --- a/messagebus/src/vespa/messagebus/blob.h +++ b/messagebus/src/vespa/messagebus/blob.h @@ -14,6 +14,7 @@ namespace mbus { **/ class Blob { + using Alloc = vespalib::alloc::Alloc; public: /** * Create a blob that will contain uninitialized memory with the @@ -22,7 +23,7 @@ public: * @param s size of the data to be created **/ Blob(uint32_t s) : - _payload(vespalib::DefaultAlloc::create(s)), + _payload(Alloc::alloc(s)), _sz(s) { } Blob(Blob && rhs) : @@ -55,11 +56,11 @@ public: **/ const char *data() const { return static_cast<const char *>(_payload.get()); } - vespalib::alloc::Alloc & payload() { return _payload; } - const vespalib::alloc::Alloc & payload() const { return _payload; } + Alloc & payload() { return _payload; } + const Alloc & payload() const { return _payload; } size_t size() const { return _sz; } private: - vespalib::alloc::Alloc _payload; + Alloc _payload; size_t _sz; }; diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 1fb7c2330c9..4f2d73dd8c9 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -12,7 +12,6 @@ import java.io.IOException; import java.io.InputStreamReader; import java.time.Duration; import java.time.Instant; -import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -23,7 +22,6 @@ import java.util.concurrent.TimeUnit; */ public class StorageMaintainer { private static final PrefixLogger NODE_ADMIN_LOGGER = PrefixLogger.getNodeAdminLogger(StorageMaintainer.class); - private static final String[] baseArguments = {"sudo", "/home/y/libexec/vespa/node-admin/maintenance.sh"}; private static final long intervalSec = 1000; private final Object monitor = new Object(); @@ -106,12 +104,12 @@ public class StorageMaintainer { DeleteOldAppData.deleteFiles(fileDistrDir.getAbsolutePath(), Duration.ofDays(31).getSeconds(), null, false); } - execute(logger, concatenateArrays(baseArguments, Maintainer.JOB_CLEAN_CORE_DUMPS)); + Maintainer.cleanCoreDumps(logger); } public void cleanNodeAdmin() { - execute(NODE_ADMIN_LOGGER, concatenateArrays(baseArguments, Maintainer.JOB_DELETE_OLD_APP_DATA)); - execute(NODE_ADMIN_LOGGER, concatenateArrays(baseArguments, Maintainer.JOB_CLEAN_HOME)); + Maintainer.deleteOldAppData(NODE_ADMIN_LOGGER); + Maintainer.cleanHome(NODE_ADMIN_LOGGER); File nodeAdminJDiskLogsPath = maintainer.pathInNodeAdminFromPathInNode(new ContainerName("node-admin"), "/home/y/logs/jdisc_core/").toFile(); @@ -120,27 +118,7 @@ public class StorageMaintainer { public void archiveNodeData(ContainerName containerName) throws IOException { PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, containerName); - execute(logger, concatenateArrays(baseArguments, Maintainer.JOB_ARCHIVE_APP_DATA, containerName.asString())); - } - - private void execute(PrefixLogger logger, String... params) { - try { - Process p = Runtime.getRuntime().exec(params); - String output = IOUtils.readAll(new InputStreamReader(p.getInputStream())); - String errors = IOUtils.readAll(new InputStreamReader(p.getErrorStream())); - - if (! output.isEmpty()) logger.info(output); - if (! errors.isEmpty()) logger.error(errors); - } catch (IOException e) { - logger.warning("Failed to execute command " + Arrays.toString(params), e); - } - } - - private static String[] concatenateArrays(String[] ar1, String... ar2) { - String[] concatenated = new String[ar1.length + ar2.length]; - System.arraycopy(ar1, 0, concatenated, 0, ar1.length); - System.arraycopy(ar2, 0, concatenated, ar1.length, ar2.length); - return concatenated; + Maintainer.archiveAppData(logger, containerName); } private static class MetricsCache { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 2bceeba2863..04937565f67 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -393,6 +393,7 @@ public class NodeAgentImpl implements NodeAgent { // If we transition from active, to not active state, unset the current metrics if (lastNodeSpec != null && lastNodeSpec.nodeState == Node.State.active && nodeSpec.nodeState != Node.State.active) { metricReceiver.unsetMetricsForContainer(hostname); + lastCpuMetric = new CpuUsageReporter(); } addDebugMessage("Loading new node spec: " + nodeSpec.toString()); lastNodeSpec = nodeSpec; @@ -460,6 +461,8 @@ public class NodeAgentImpl implements NodeAgent { } if (nodeSpec == null || nodeSpec.nodeState != Node.State.active) return; + final Optional<Container> container = dockerOperations.getContainer(nodeSpec.hostname); + if ( ! container.isPresent() || ! container.get().isRunning ) return; Docker.ContainerStats stats = dockerOperations.getContainerStats(nodeSpec.containerName); Dimensions.Builder dimensionsBuilder = new Dimensions.Builder() diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/Maintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/Maintainer.java index 991cf6b31df..02a05f9fa9d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/Maintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/maintenance/Maintainer.java @@ -1,7 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.maintenance; +import com.yahoo.io.IOUtils; +import com.yahoo.log.LogSetup; import com.yahoo.vespa.hosted.dockerapi.ContainerName; +import com.yahoo.vespa.hosted.node.admin.util.PrefixLogger; import io.airlift.airline.Arguments; import io.airlift.airline.Cli; import io.airlift.airline.Command; @@ -11,6 +14,7 @@ import io.airlift.airline.ParseOptionMissingException; import java.io.File; import java.io.IOException; +import java.io.InputStreamReader; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -21,6 +25,7 @@ import java.time.Instant; import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.TimeZone; import java.util.logging.Logger; import java.util.regex.Pattern; @@ -48,6 +53,8 @@ public class Maintainer { @SuppressWarnings("unchecked") public static void main(String[] args) { + LogSetup.initVespaLogging(Maintainer.class.getSimpleName().toLowerCase()); + Cli.CliBuilder<Runnable> builder = Cli.<Runnable>builder("maintainer.jar") .withDescription("This tool makes it easy to delete old log files and other node-admin app data.") .withDefaultCommand(Help.class) @@ -66,6 +73,48 @@ public class Maintainer { } } + public static void cleanCoreDumps(PrefixLogger logger) { + executeMaintainer(logger, JOB_CLEAN_CORE_DUMPS); + } + + public static void deleteOldAppData(PrefixLogger logger) { + executeMaintainer(logger, JOB_DELETE_OLD_APP_DATA); + } + + public static void cleanHome(PrefixLogger logger) { + executeMaintainer(logger, JOB_CLEAN_HOME); + } + + public static void archiveAppData(PrefixLogger logger, ContainerName containerName) { + executeMaintainer(logger, JOB_ARCHIVE_APP_DATA, containerName.asString()); + } + + private static void executeMaintainer(PrefixLogger logger, String... params) { + String[] baseArguments = {"sudo", "/home/y/libexec/vespa/node-admin/maintenance.sh"}; + String[] args = concatenateArrays(baseArguments, params); + ProcessBuilder processBuilder = new ProcessBuilder(args); + Map<String, String> env = processBuilder.environment(); + env.put("VESPA_SERVICE_NAME", "maintainer"); + + try { + Process process = processBuilder.start(); + String output = IOUtils.readAll(new InputStreamReader(process.getInputStream())); + String errors = IOUtils.readAll(new InputStreamReader(process.getErrorStream())); + + if (! output.isEmpty()) logger.info(output); + if (! errors.isEmpty()) logger.error(errors); + } catch (IOException e) { + logger.warning("Failed to execute command " + Arrays.toString(args), e); + } + } + + public static String[] concatenateArrays(String[] ar1, String... ar2) { + String[] concatenated = new String[ar1.length + ar2.length]; + System.arraycopy(ar1, 0, concatenated, 0, ar1.length); + System.arraycopy(ar2, 0, concatenated, ar1.length, ar2.length); + return concatenated; + } + @Command(name = JOB_DELETE_OLD_APP_DATA, description = "Deletes old app data") public static class DeleteOldAppDataArguments implements Runnable { @Override diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index ddda3fe13e3..0c9d6881a0d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -419,6 +419,9 @@ public class NodeAgentImplTest { final ContainerName containerName = new ContainerName("cont-name"); when(dockerOperations.getContainerStats(eq(containerName))).thenReturn(stats); + when(dockerOperations.getContainer(eq(hostName))) + .thenReturn(Optional.of(new Container(hostName, new DockerImage("wantedDockerImage"), containerName, true))); + Optional<String> version = Optional.of("1.2.3"); ContainerNodeSpec.Owner owner = new ContainerNodeSpec.Owner("tester", "testapp", "testinstance"); ContainerNodeSpec.Membership membership = new ContainerNodeSpec.Membership("clustType", "clustId", "grp", 3, false); diff --git a/persistence/src/tests/CMakeLists.txt b/persistence/src/tests/CMakeLists.txt index e547c4e72eb..934bc54eb9c 100644 --- a/persistence/src/tests/CMakeLists.txt +++ b/persistence/src/tests/CMakeLists.txt @@ -5,7 +5,5 @@ vespa_add_executable(persistence_testrunner_app TEST DEPENDS persistence_testdummyimpl persistence_testspi - persistence - persistence_persistence_conformancetest ) vespa_add_test(NAME persistence_testrunner_app COMMAND persistence_testrunner_app) diff --git a/persistence/src/tests/dummyimpl/CMakeLists.txt b/persistence/src/tests/dummyimpl/CMakeLists.txt index 66758e8a0b4..6dabdd628fd 100644 --- a/persistence/src/tests/dummyimpl/CMakeLists.txt +++ b/persistence/src/tests/dummyimpl/CMakeLists.txt @@ -3,12 +3,13 @@ vespa_add_library(persistence_testdummyimpl SOURCES dummyimpltest.cpp DEPENDS + persistence_persistence_conformancetest + persistence ) vespa_add_executable(persistence_dummypersistence_test_app TEST SOURCES dummypersistence_test.cpp DEPENDS persistence - persistence_persistence_conformancetest ) vespa_add_test(NAME persistence_dummypersistence_test_app COMMAND persistence_dummypersistence_test_app) diff --git a/persistence/src/tests/spi/CMakeLists.txt b/persistence/src/tests/spi/CMakeLists.txt index d23c9c44209..236c8da2795 100644 --- a/persistence/src/tests/spi/CMakeLists.txt +++ b/persistence/src/tests/spi/CMakeLists.txt @@ -3,4 +3,6 @@ vespa_add_library(persistence_testspi SOURCES clusterstatetest.cpp DEPENDS + persistence_persistence_conformancetest + persistence ) diff --git a/persistence/src/vespa/persistence/CMakeLists.txt b/persistence/src/vespa/persistence/CMakeLists.txt index a45ca2e25f4..e93eb012fd5 100644 --- a/persistence/src/vespa/persistence/CMakeLists.txt +++ b/persistence/src/vespa/persistence/CMakeLists.txt @@ -12,4 +12,6 @@ vespa_add_library(persistence_persistence_conformancetest $<TARGET_OBJECTS:persistence_conformancetest_lib> INSTALL lib64 DEPENDS + persistence + vdstestlib ) diff --git a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp index 1983ff7a318..f57e464fae6 100644 --- a/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp +++ b/searchcore/src/apps/vespa-gen-testdocs/vespa-gen-testdocs.cpp @@ -17,7 +17,6 @@ typedef vespalib::hash_set<uint32_t> UIntSet; typedef std::vector<vespalib::string> StringArray; typedef std::shared_ptr<StringArray> StringArraySP; using namespace vespalib::alloc; -using vespalib::DefaultAlloc; using vespalib::string; void @@ -49,7 +48,7 @@ shafile(const string &baseDir, string fullFile(prependBaseDir(baseDir, file)); FastOS_File f; std::ostringstream os; - Alloc buf = DefaultAlloc::create(65536, MemoryAllocator::HUGEPAGE_SIZE, 0x1000); + Alloc buf = Alloc::alloc(65536, MemoryAllocator::HUGEPAGE_SIZE, 0x1000); f.EnableDirectIO(); bool openres = f.OpenReadOnly(fullFile.c_str()); if (!openres) { diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp index 781da158254..2ed3fbbf0fe 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.cpp @@ -27,98 +27,194 @@ LOG_SETUP(".proton.matching.match_thread"); namespace proton { namespace matching { +using search::queryeval::OptimizedAndNotForBlackListing; +using search::queryeval::SearchIterator; +using search::fef::TermFieldHandle; +using search::fef::MatchData; +using search::fef::RankProgram; + namespace { -using search::fef::FeatureHandle; -using search::fef::IllegalHandle; -using search::fef::RankProgram; +struct WaitTimer { + double &wait_time_s; + fastos::StopWatch wait_time; + WaitTimer(double &wait_time_s_in) + : wait_time_s(wait_time_s_in), wait_time() + { + wait_time.start(); + } + void done() { + wait_time.stop(); + wait_time_s += wait_time.elapsed().sec(); + } +}; + +class FastSeekWrapper +{ +private: + typedef search::queryeval::SearchIterator SearchIterator; +public: + FastSeekWrapper(SearchIterator::UP iterator) + { + reset(iterator.release()); + } + void initRange(uint32_t begin_id, uint32_t end_id) { + _search->initRange(begin_id, end_id); + } + uint32_t seekFirst(uint32_t docId) { + return _search->seekFirst(docId); + } + uint32_t seekNext(uint32_t docId) { + return _search->seekFast(docId); + } + vespalib::string asString() const { + return _search->asString(); + } + void unpack(uint32_t docId) { + _search->unpack(docId); + } + void reset(SearchIterator * search) { + _search.reset(&dynamic_cast<OptimizedAndNotForBlackListing &>(*search)); + } + OptimizedAndNotForBlackListing * release() { + return _search.release(); + } + FastSeekWrapper * operator ->() { return this; } +private: + std::unique_ptr<OptimizedAndNotForBlackListing> _search; +}; const double *get_score_feature(const RankProgram &rankProgram) { std::vector<vespalib::string> featureNames; - std::vector<FeatureHandle> featureHandles; + std::vector<search::fef::FeatureHandle> featureHandles; rankProgram.get_seed_handles(featureNames, featureHandles); assert(featureNames.size() == 1); assert(featureHandles.size() == 1); return rankProgram.match_data().resolveFeature(featureHandles.front()); } +} // namespace proton::matching::<unnamed> + +//----------------------------------------------------------------------------- + +MatchThread::Context::Context(double rankDropLimit, MatchTools & matchTools, RankProgram & ranking, HitCollector & hits, + uint32_t num_threads) : + matches(0), + _matches_limit(matchTools.match_limiter().sample_hits_per_thread(num_threads)), + _score_feature(get_score_feature(ranking)), + _ranking(ranking), + _rankDropLimit(rankDropLimit), + _hits(hits), + _doom(matchTools.doom()), + _limiter(matchTools.match_limiter()) +{ } -using vespalib::Doom; -using search::queryeval::OptimizedAndNotForBlackListing; +void +MatchThread::Context::rankHit(uint32_t docId) { + _ranking.run(docId); + double score = *_score_feature; + // convert NaN and Inf scores to -Inf + if (__builtin_expect(std::isnan(score) || std::isinf(score), false)) { + score = -HUGE_VAL; + } + // invert test since default drop limit is -NaN (keep all hits) + if (!(score <= _rankDropLimit)) { + _hits.addHit(docId, score); + } +} + +//----------------------------------------------------------------------------- + +double +MatchThread::estimate_match_frequency(uint32_t matches, uint32_t searchedSoFar) +{ + IMatchLoopCommunicator::Matches my_matches(matches, searchedSoFar); + WaitTimer count_matches_timer(wait_time_s); + double match_freq = communicator.estimate_match_frequency(my_matches); + count_matches_timer.done(); + return match_freq; +} + +template <typename IteratorT> +void +MatchThread::maybe_limit(MaybeMatchPhaseLimiter & limiter, IteratorT & search, uint32_t matches, uint32_t docId, uint32_t endId) +{ + const uint32_t local_todo = (endId - docId - 1); + const size_t searchedSoFar = (scheduler.total_size(thread_id) - local_todo); + double match_freq = estimate_match_frequency(matches, searchedSoFar); + const size_t global_todo = scheduler.unassigned_size(); + search = limiter.maybe_limit(std::move(search), match_freq, matchParams.numDocs); + size_t left = local_todo + (global_todo / num_threads); + limiter.updateDocIdSpaceEstimate(searchedSoFar, left); + LOG(debug, "Limit=%d has been reached at docid=%d which is after %zu docs.", + matches, docId, (scheduler.total_size(thread_id) - local_todo)); + LOG(debug, "SearchIterator after limiter: %s", search->asString().c_str()); +} + +template <> +void +MatchThread::maybe_limit(MaybeMatchPhaseLimiter &, FastSeekWrapper &, uint32_t, uint32_t, uint32_t) +{ + abort(); // We cannot replace the iterator if we inline the loop. +} + +bool +MatchThread::try_share(DocidRange &docid_range, uint32_t next_docid) { + DocidRange todo(next_docid, docid_range.end); + DocidRange my_work = scheduler.share_range(thread_id, todo); + if (my_work.end < todo.end) { + docid_range = my_work; + return true; + } + return false; +} + +template <typename IteratorT, bool do_rank, bool do_limit, bool do_share_work> +void +MatchThread::inner_match_loop(Context & context, IteratorT & search, DocidRange docid_range) +{ + search->initRange(docid_range.begin, docid_range.end); + uint32_t docId = search->seekFirst(docid_range.begin); + while ((docId < docid_range.end) && !context.doom()) { + if (do_rank) { + search->unpack(docId); + context.rankHit(docId); + } else { + context.addHit(docId); + } + context.matches++; + if (do_limit && context.isAtLimit()) { + maybe_limit(context.limiter(), search, context.matches, docId, docid_range.end); + docId = search->seekFirst(docId + 1); + } else if (do_share_work && any_idle() && try_share(docid_range, docId + 1)) { + search->initRange(docid_range.begin, docid_range.end); + docId = search->seekFirst(docid_range.begin); + } else { + docId = search->seekNext(docId + 1); + } + } +} template <typename IteratorT, bool do_rank, bool do_limit, bool do_share_work> void MatchThread::match_loop(MatchTools &matchTools, IteratorT search, RankProgram &ranking, HitCollector &hits) { - const Doom &doom = matchTools.doom(); - const double *score_feature = do_rank ? get_score_feature(ranking) : nullptr; - uint32_t matches = 0; - uint32_t matches_limit = (do_limit) ? matchTools.match_limiter().sample_hits_per_thread(num_threads) : 0; - IdleObserver idle_observer = scheduler.make_idle_observer(); + Context context(matchParams.rankDropLimit, matchTools, ranking, hits, num_threads); for (DocidRange docid_range = scheduler.first_range(thread_id); !docid_range.empty(); docid_range = scheduler.next_range(thread_id)) { - search->initRange(docid_range.begin, docid_range.end); - uint32_t docId = search->seekFirst(docid_range.begin); - while ((docId < docid_range.end) && !doom.doom()) { - if (do_rank) { - search->unpack(docId); - ranking.run(docId); - double score = *score_feature; - // convert NaN and Inf scores to -Inf - if (__builtin_expect(std::isnan(score) || std::isinf(score), false)) { - score = -HUGE_VAL; - } - // invert test since default drop limit is -NaN (keep all hits) - if (!(score <= matchParams.rankDropLimit)) { - hits.addHit(docId, score); - } - } else { - hits.addHit(docId, 0.0); - } - ++matches; - if (do_limit && matches == matches_limit) { - const size_t local_todo = (docid_range.end - docId - 1); - const size_t searchedSoFar = (scheduler.total_size(thread_id) - local_todo); - IMatchLoopCommunicator::Matches my_matches(matches, searchedSoFar); - WaitTimer count_matches_timer(wait_time_s); - double match_freq = communicator.estimate_match_frequency(my_matches); - const size_t global_todo = scheduler.unassigned_size(); - count_matches_timer.done(); - search.reset(matchTools.match_limiter().maybe_limit(SearchIterator::UP(search.release()), - match_freq, - matchParams.numDocs).release()); - matchTools.match_limiter().updateDocIdSpaceEstimate(searchedSoFar, local_todo + (global_todo / num_threads)); - LOG(debug, "Limit=%d has been reached at docid=%d which is after %zu docs.", - matches, docId, searchedSoFar); - LOG(debug, "SearchIterator after limiter: %s", search->asString().c_str()); - docId = search->seekFirst(docId + 1); - } else if (do_share_work && (idle_observer.get() > 0)) { - DocidRange todo(docId + 1, docid_range.end); - DocidRange my_work = scheduler.share_range(thread_id, todo); - if (my_work.end < todo.end) { - docid_range = my_work; - search->initRange(docid_range.begin, docid_range.end); - docId = search->seekFirst(docid_range.begin); - } else { - docId = search->seekNext(docId + 1); - } - } else { - docId = search->seekNext(docId + 1); - } - } + inner_match_loop<IteratorT, do_rank, do_limit, do_share_work>(context, search, docid_range); } - if (do_limit && matches < matches_limit) { - IMatchLoopCommunicator::Matches my_matches(matches, scheduler.total_size(thread_id)); - WaitTimer count_matches_timer(wait_time_s); + uint32_t matches = context.matches; + if (do_limit && context.isBelowLimit()) { + const size_t searchedSoFar = scheduler.total_size(thread_id); LOG(debug, "Limit not reached (had %d) at docid=%d which is after %zu docs.", - matches, scheduler.total_span(thread_id).end, scheduler.total_size(thread_id)); - communicator.estimate_match_frequency(my_matches); // for other threads - matchTools.match_limiter().updateDocIdSpaceEstimate(scheduler.total_size(thread_id), 0); - count_matches_timer.done(); + matches, scheduler.total_span(thread_id).end, searchedSoFar); + estimate_match_frequency(matches, searchedSoFar); + context.limiter().updateDocIdSpaceEstimate(searchedSoFar, 0); } thread_stats.docsMatched(matches); if (do_rank) { @@ -131,7 +227,7 @@ void MatchThread::match_loop_helper_2(MatchTools &matchTools, IteratorT search, RankProgram &ranking, HitCollector &hits) { - if (scheduler.make_idle_observer().is_always_zero()) { + if (idle_observer.is_always_zero()) { match_loop<IteratorT, do_rank, do_limit, false>(matchTools, std::move(search), ranking, hits); } else { match_loop<IteratorT, do_rank, do_limit, true>(matchTools, std::move(search), ranking, hits); @@ -150,69 +246,7 @@ MatchThread::match_loop_helper(MatchTools &matchTools, IteratorT search, } } -using search::fef::TermFieldHandle; -using search::fef::MatchData; - -class FastSeekWrapper -{ -private: - typedef search::queryeval::SearchIterator SearchIterator; -public: - FastSeekWrapper(SearchIterator::UP iterator) - { - reset(iterator.release()); - } - void initRange(uint32_t begin_id, uint32_t end_id) { - _search->initRange(begin_id, end_id); - } - uint32_t seekFirst(uint32_t docId) { - return _search->seekFirst(docId); - } - uint32_t seekNext(uint32_t docId) { - return _search->seekFast(docId); - } - vespalib::string asString() const { - return _search->asString(); - } - void unpack(uint32_t docId) { - _search->unpack(docId); - } - void reset(SearchIterator * search) { - _search.reset(&dynamic_cast<OptimizedAndNotForBlackListing &>(*search)); - } - OptimizedAndNotForBlackListing * release() { - return _search.release(); - } - FastSeekWrapper * operator ->() { return this; } -private: - std::unique_ptr<OptimizedAndNotForBlackListing> _search; -}; - -MatchThread::MatchThread(size_t thread_id_in, - size_t num_threads_in, - const MatchParams &mp, - const MatchToolsFactory &mtf, - IMatchLoopCommunicator &com, - DocidRangeScheduler &sched, - ResultProcessor &rp, - vespalib::DualMergeDirector &md, - uint32_t distributionKey) : - thread_id(thread_id_in), - num_threads(num_threads_in), - matchParams(mp), - matchToolsFactory(mtf), - communicator(com), - scheduler(sched), - _distributionKey(distributionKey), - resultProcessor(rp), - mergeDirector(md), - resultContext(), - thread_stats(), - total_time_s(0.0), - match_time_s(0.0), - wait_time_s(0.0) -{ -} +//----------------------------------------------------------------------------- search::ResultSet::UP MatchThread::findMatches(MatchTools &matchTools) @@ -228,9 +262,9 @@ MatchThread::findMatches(MatchTools &matchTools) match_loop_helper<SearchIterator::UP, true>(matchTools, std::move(search), *ranking, hits); } else { if ((dynamic_cast<const OptimizedAndNotForBlackListing *>(search.get()) != 0) && - ! matchTools.match_limiter().is_enabled()) // We cannot replace the iterator if we inline the loop. + ! matchTools.match_limiter().is_enabled()) // We cannot replace the iterator if we inline the loop. { - match_loop_helper<FastSeekWrapper, false>(matchTools, FastSeekWrapper(std::move(search)), *ranking, hits); + match_loop_helper_2<FastSeekWrapper, false, false>(matchTools, FastSeekWrapper(std::move(search)), *ranking, hits); } else { match_loop_helper<SearchIterator::UP, false>(matchTools, std::move(search), *ranking, hits); } @@ -318,6 +352,35 @@ MatchThread::processResult(const Doom & doom, } } +//----------------------------------------------------------------------------- + +MatchThread::MatchThread(size_t thread_id_in, + size_t num_threads_in, + const MatchParams &mp, + const MatchToolsFactory &mtf, + IMatchLoopCommunicator &com, + DocidRangeScheduler &sched, + ResultProcessor &rp, + vespalib::DualMergeDirector &md, + uint32_t distributionKey) : + thread_id(thread_id_in), + num_threads(num_threads_in), + matchParams(mp), + matchToolsFactory(mtf), + communicator(com), + scheduler(sched), + idle_observer(scheduler.make_idle_observer()), + _distributionKey(distributionKey), + resultProcessor(rp), + mergeDirector(md), + resultContext(), + thread_stats(), + total_time_s(0.0), + match_time_s(0.0), + wait_time_s(0.0) +{ +} + void MatchThread::run() { diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h index 5b298695253..478ee375d38 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_thread.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_thread.h @@ -24,11 +24,12 @@ namespace matching { class MatchThread : public vespalib::Runnable { public: - typedef std::unique_ptr<MatchThread> UP; - typedef search::queryeval::SearchIterator SearchIterator; - typedef search::fef::MatchData MatchData; - typedef search::queryeval::HitCollector HitCollector; - typedef search::fef::RankProgram RankProgram; + using UP = std::unique_ptr<MatchThread>; + using SearchIterator = search::queryeval::SearchIterator; + using MatchData = search::fef::MatchData; + using HitCollector = search::queryeval::HitCollector; + using RankProgram = search::fef::RankProgram; + using Doom = vespalib::Doom; private: size_t thread_id; @@ -37,6 +38,7 @@ private: const MatchToolsFactory &matchToolsFactory; IMatchLoopCommunicator &communicator; DocidRangeScheduler &scheduler; + IdleObserver idle_observer; uint32_t _distributionKey; ResultProcessor &resultProcessor; vespalib::DualMergeDirector &mergeDirector; @@ -46,38 +48,50 @@ private: double match_time_s; double wait_time_s; - struct WaitTimer { - double &wait_time_s; - fastos::StopWatch wait_time; - WaitTimer(double &wait_time_s_in) - : wait_time_s(wait_time_s_in), wait_time() - { - wait_time.start(); - } - void done() { - wait_time.stop(); - wait_time_s += wait_time.elapsed().sec(); - } + class Context { + public: + Context(double rankDropLimit, MatchTools &matchTools, RankProgram & ranking, HitCollector & hits, + uint32_t num_threads) __attribute__((noinline)); + void rankHit(uint32_t docId); + void addHit(uint32_t docId) { _hits.addHit(docId, 0.0); } + bool isBelowLimit() const { return matches < _matches_limit; } + bool isAtLimit() const { return matches == _matches_limit; } + bool doom() const { return _doom.doom(); } + MaybeMatchPhaseLimiter & limiter() { return _limiter; } + uint32_t matches; + private: + uint32_t _matches_limit; + const double * _score_feature; + RankProgram & _ranking; + double _rankDropLimit; + HitCollector & _hits; + const Doom & _doom; + MaybeMatchPhaseLimiter & _limiter; }; + double estimate_match_frequency(uint32_t matches, uint32_t searchedSoFar) __attribute__((noinline)); + + template <typename IteratorT> + void maybe_limit(MaybeMatchPhaseLimiter & limiter, IteratorT & search, uint32_t matches, uint32_t docId, uint32_t endId) __attribute__((noinline)); + + bool any_idle() const { return (idle_observer.get() > 0); } + bool try_share(DocidRange &docid_range, uint32_t next_docid) __attribute__((noinline)); + template <typename IteratorT, bool do_rank, bool do_limit, bool do_share_work> - void match_loop(MatchTools &matchTools, IteratorT search, - RankProgram &ranking, HitCollector &hits) __attribute__((noinline)); + void inner_match_loop(Context & params, IteratorT & search, DocidRange docid_range) __attribute__((noinline)); + + template <typename IteratorT, bool do_rank, bool do_limit, bool do_share_work> + void match_loop(MatchTools &matchTools, IteratorT search, RankProgram &ranking, HitCollector &hits) __attribute__((noinline)); template <typename IteratorT, bool do_rank, bool do_limit> - void match_loop_helper_2(MatchTools &matchTools, IteratorT search, - RankProgram &ranking, HitCollector &hits); + void match_loop_helper_2(MatchTools &matchTools, IteratorT search, RankProgram &ranking, HitCollector &hits); template <typename IteratorT, bool do_rank> - void match_loop_helper(MatchTools &matchTools, IteratorT search, - RankProgram &ranking, HitCollector &hits); + void match_loop_helper(MatchTools &matchTools, IteratorT search, RankProgram &ranking, HitCollector &hits); search::ResultSet::UP findMatches(MatchTools &matchTools); - void processResult(const vespalib::Doom & doom, - search::ResultSet::UP result, - ResultProcessor::Context &context); - + void processResult(const Doom & doom, search::ResultSet::UP result, ResultProcessor::Context &context); public: MatchThread(size_t thread_id_in, size_t num_threads_in, @@ -95,4 +109,3 @@ public: } // namespace proton::matching } // namespace proton - diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index f576e53e96b..b48ba6e50e0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -865,7 +865,7 @@ Proton::prepareRestart() std::make_shared<PrepareRestartFlushStrategy>( createPrepareRestartConfig(configSnapshot->getProtonConfig())); _flushEngine->setStrategy(strategy); - return false; + return true; } void diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp index 5433b4aac20..2e2a7f13fae 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp @@ -266,7 +266,7 @@ RPCHooksBase::triggerFlush(FRT_RPCRequest *req) { if (_proton.triggerFlush()) { req->GetReturn()->AddInt8(1); - LOG(info, "RPCHooksBase::Flush finished sucessfully"); + LOG(info, "RPCHooksBase::Flush finished successfully"); } else { req->GetReturn()->AddInt8(0); LOG(warning, "RPCHooksBase::Flush failed"); @@ -279,7 +279,7 @@ RPCHooksBase::prepareRestart(FRT_RPCRequest *req) { if (_proton.prepareRestart()) { req->GetReturn()->AddInt8(1); - LOG(info, "RPCHooksBase::prepareRestart finished sucessfully"); + LOG(info, "RPCHooksBase::prepareRestart finished successfully"); } else { req->GetReturn()->AddInt8(0); LOG(warning, "RPCHooksBase::prepareRestart failed"); @@ -447,7 +447,7 @@ void RPCHooksBase::wipeHistory(FRT_RPCRequest *req) { _proton.wipeHistory(); - LOG(info, "RPCHooksBase::wipeHistory finished sucessfully"); + LOG(info, "RPCHooksBase::wipeHistory finished successfully"); req->Return(); } diff --git a/searchcorespi/src/tests/plugin/CMakeLists.txt b/searchcorespi/src/tests/plugin/CMakeLists.txt index 333f927c3d2..ee449a8a3ff 100644 --- a/searchcorespi/src/tests/plugin/CMakeLists.txt +++ b/searchcorespi/src/tests/plugin/CMakeLists.txt @@ -3,11 +3,13 @@ vespa_add_library(searchcorespi_tplugin SOURCES plugin.cpp DEPENDS + searchcorespi ) vespa_add_library(searchcorespi_illegal-plugin SOURCES empty.cpp DEPENDS + searchcorespi ) vespa_add_executable(searchcorespi_plugin_test_app TEST SOURCES diff --git a/searchlib/src/tests/datastore/logdatastore_test.cpp b/searchlib/src/tests/datastore/logdatastore_test.cpp index 00d4568df3e..7c558c2119d 100644 --- a/searchlib/src/tests/datastore/logdatastore_test.cpp +++ b/searchlib/src/tests/datastore/logdatastore_test.cpp @@ -20,7 +20,6 @@ using document::BucketId; using namespace search::docstore; using namespace search; using namespace vespalib::alloc; -using vespalib::DefaultAlloc; using search::index::DummyFileHeaderContext; class MyTlSyncer : public transactionlog::SyncProxy { @@ -146,7 +145,7 @@ TEST("test that DirectIOPadding works accordng to spec") { FastOS_File file("directio.test"); file.EnableDirectIO(); EXPECT_TRUE(file.OpenReadWrite()); - Alloc buf(DefaultAlloc::create(FILE_SIZE, MemoryAllocator::HUGEPAGE_SIZE, 4096)); + Alloc buf(Alloc::alloc(FILE_SIZE, MemoryAllocator::HUGEPAGE_SIZE, 4096)); memset(buf.get(), 'a', buf.size()); EXPECT_EQUAL(FILE_SIZE, file.Write2(buf.get(), FILE_SIZE)); size_t padBefore(0); diff --git a/searchlib/src/vespa/searchlib/attribute/loadedstringvalue.cpp b/searchlib/src/vespa/searchlib/attribute/loadedstringvalue.cpp index 478b7a5e7a9..954e4342466 100644 --- a/searchlib/src/vespa/searchlib/attribute/loadedstringvalue.cpp +++ b/searchlib/src/vespa/searchlib/attribute/loadedstringvalue.cpp @@ -4,7 +4,7 @@ #include "loadedstringvalue.h" using vespalib::Array; -using vespalib::alloc::MMapAllocFactory; +using vespalib::alloc::Alloc; namespace search { @@ -13,7 +13,7 @@ namespace attribute { void sortLoadedByValue(LoadedStringVectorReal &loaded) { - Array<unsigned> radixScratchPad(loaded.size(), MMapAllocFactory::create()); + Array<unsigned> radixScratchPad(loaded.size(), Alloc::allocMMap()); for(size_t i(0), m(loaded.size()); i < m; i++) { loaded[i].prepareRadixSort(); } diff --git a/searchlib/src/vespa/searchlib/btree/bufferstate.cpp b/searchlib/src/vespa/searchlib/btree/bufferstate.cpp index 36e63300034..31e3a6c3e3f 100644 --- a/searchlib/src/vespa/searchlib/btree/bufferstate.cpp +++ b/searchlib/src/vespa/searchlib/btree/bufferstate.cpp @@ -3,7 +3,6 @@ #include "bufferstate.h" #include <limits> -using vespalib::DefaultAlloc; using vespalib::alloc::Alloc; namespace search { @@ -125,7 +124,7 @@ BufferState::BufferState(void) _typeId(0), _clusterSize(0), _compacting(false), - _buffer(DefaultAlloc::create()) + _buffer(Alloc::alloc()) { } @@ -216,7 +215,7 @@ BufferState::onFree(void *&buffer) assert(_deadElems <= _usedElems); assert(_holdElems == _usedElems - _deadElems); _typeHandler->destroyElements(buffer, _usedElems); - DefaultAlloc::create().swap(_buffer); + Alloc::alloc().swap(_buffer); _typeHandler->onFree(_usedElems); buffer = NULL; _usedElems = 0; diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp index 255ee64633d..5c4477b9f97 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp @@ -11,14 +11,13 @@ using vespalib::nbostream; using vespalib::GenerationHeldBase; using vespalib::GenerationHeldAlloc; using vespalib::GenerationHolder; -using vespalib::DefaultAlloc; void AllocatedBitVector::alloc() { uint32_t words = capacityWords(); words += (-words & 15); // Pad to 64 byte alignment const size_t sz(words * sizeof(Word)); - DefaultAlloc::create(sz).swap(_alloc); + Alloc::alloc(sz).swap(_alloc); assert(_alloc.size()/sizeof(Word) >= words); // Clear padding memset(static_cast<char *>(_alloc.get()) + sizeBytes(), 0, sz - sizeBytes()); diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index 441351ab724..4f04a557085 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -14,7 +14,7 @@ using vespalib::make_string; using vespalib::IllegalArgumentException; using vespalib::hwaccelrated::IAccelrated; using vespalib::Optimized; -using vespalib::DefaultAlloc; +using vespalib::alloc::Alloc; namespace { @@ -323,7 +323,7 @@ BitVector::create(Index numberOfElements, size_t vectorsize = getFileBytes(numberOfElements); file.DirectIOPadding(offset, vectorsize, padbefore, padafter); assert((padbefore & (getAlignment() - 1)) == 0); - AllocatedBitVector::Alloc alloc = DefaultAlloc::create(padbefore + vectorsize + padafter, 0x1000000, 0x1000); + AllocatedBitVector::Alloc alloc = Alloc::alloc(padbefore + vectorsize + padafter, 0x1000000, 0x1000); void * alignedBuffer = alloc.get(); file.ReadBuf(alignedBuffer, alloc.size(), offset - padbefore); bv.reset(new AllocatedBitVector(numberOfElements, std::move(alloc), padbefore)); diff --git a/searchlib/src/vespa/searchlib/common/partialbitvector.cpp b/searchlib/src/vespa/searchlib/common/partialbitvector.cpp index 59130ac7ec9..ba165421a41 100644 --- a/searchlib/src/vespa/searchlib/common/partialbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/partialbitvector.cpp @@ -5,13 +5,13 @@ #include <vespa/fastos/fastos.h> #include "partialbitvector.h" -using vespalib::DefaultAlloc; - namespace search { +using vespalib::alloc::Alloc; + PartialBitVector::PartialBitVector(Index start, Index end) : BitVector(), - _alloc(DefaultAlloc::create(numActiveBytes(start, end), 0x1000000, 0x1000)) + _alloc(Alloc::alloc(numActiveBytes(start, end), 0x1000000, 0x1000)) { init(_alloc.get(), start, end); clear(); diff --git a/searchlib/src/vespa/searchlib/common/resultset.cpp b/searchlib/src/vespa/searchlib/common/resultset.cpp index cdbfebdc1a9..0544e54b59b 100644 --- a/searchlib/src/vespa/searchlib/common/resultset.cpp +++ b/searchlib/src/vespa/searchlib/common/resultset.cpp @@ -6,7 +6,6 @@ #include <vespa/searchlib/common/resultset.h> #include <vespa/searchlib/common/bitvector.h> -using vespalib::DefaultAlloc; using vespalib::alloc::Alloc; namespace search { @@ -51,7 +50,7 @@ void ResultSet::allocArray(unsigned int arrayAllocated) { if (arrayAllocated > 0) { - DefaultAlloc::create(arrayAllocated * sizeof(RankedHit), MMAP_LIMIT).swap(_rankedHitsArray); + Alloc::alloc(arrayAllocated * sizeof(RankedHit), MMAP_LIMIT).swap(_rankedHitsArray); } else { Alloc().swap(_rankedHitsArray); } @@ -99,7 +98,7 @@ ResultSet::mergeWithBitOverflow(void) uint32_t bidx = bitVector->getFirstTrueBit(); uint32_t actualHits = getNumHits(); - Alloc newHitsAlloc = DefaultAlloc::create(actualHits*sizeof(RankedHit), MMAP_LIMIT); + Alloc newHitsAlloc = Alloc::alloc(actualHits*sizeof(RankedHit), MMAP_LIMIT); RankedHit *newHitsArray = static_cast<RankedHit *>(newHitsAlloc.get()); RankedHit * tgtA = newHitsArray; diff --git a/searchlib/src/vespa/searchlib/common/sortresults.cpp b/searchlib/src/vespa/searchlib/common/sortresults.cpp index 76109f3ec42..0b430369e96 100644 --- a/searchlib/src/vespa/searchlib/common/sortresults.cpp +++ b/searchlib/src/vespa/searchlib/common/sortresults.cpp @@ -16,9 +16,7 @@ using search::common::SortSpec; using search::common::SortInfo; using search::attribute::IAttributeContext; using search::attribute::IAttributeVector; - -using vespalib::DefaultAlloc; - +using vespalib::alloc::Alloc; namespace { constexpr size_t MMAP_LIMIT = 0x2000000; @@ -29,8 +27,7 @@ class RadixHelper public: typedef vespalib::convertForSort<T, true> C; inline typename C::UIntType - operator()(typename C::InputType v) const - { + operator()(typename C::InputType v) const { return C::convert(v); } }; @@ -46,12 +43,10 @@ FastS_insertion_sort(RankedHit a[], uint32_t n) typedef RadixHelper<search::HitRank> RT; RT R; - for (i=1; i<n ; i++) - { + for (i=1; i<n ; i++) { swap = a[i]; j = i; - while (R(swap._rankValue) > R(a[j-1]._rankValue)) - { + while (R(swap._rankValue) > R(a[j-1]._rankValue)) { a[j] = a[j-1]; if (!(--j)) break;; } @@ -87,24 +82,20 @@ FastS_radixsort(RankedHit a[], uint32_t n, uint32_t ntop) sorted = (cnt[0]==n); ptr[0] = n-cnt[0]; last[0] = n; - for(i=1; i<256; i++) - { + for(i=1; i<256; i++) { ptr[i] = (last[i]=ptr[i-1]) - cnt[i]; sorted |= (cnt[i]==n); } - if (!sorted) - { + if (!sorted) { // Go through all permutation cycles until all // elements are moved or found to be already in place i = 255; remain = n; - while(remain>0) - { + while(remain>0) { // Find first uncompleted class - while(ptr[i]==last[i]) - { + while(ptr[i]==last[i]) { i--; } @@ -117,10 +108,8 @@ FastS_radixsort(RankedHit a[], uint32_t n, uint32_t ntop) k = (R(swap._rankValue) >> SHIFT) & 0xFF; // Swap into correct class until cycle completed - if (i!=k) - { - do - { + if (i!=k) { + do { temp = a[ptr[k]]; a[ptr[k]++] = swap; k = (R((swap = temp)._rankValue) >> SHIFT) & 0xFF; @@ -137,10 +126,9 @@ FastS_radixsort(RankedHit a[], uint32_t n, uint32_t ntop) return; } - if (SHIFT>0) - { + if (SHIFT>0) { // Sort on next key - for(i=0; i<256 ; i++) + for(i=0; i<256 ; i++) { if ((last[i]-cnt[i])<ntop) { if (cnt[i]>INSERT_SORT_LEVEL) { if (last[i]<ntop) { @@ -153,6 +141,7 @@ FastS_radixsort(RankedHit a[], uint32_t n, uint32_t ntop) } else if (cnt[i]>1) { FastS_insertion_sort(&a[last[i]-cnt[i]], cnt[i]); } + } } } } @@ -428,12 +417,10 @@ FastS_insertion_sort(T a[], uint32_t n, Compare *compobj) uint32_t i, j; T swap; - for (i=1; i<n ; i++) - { + for (i=1; i<n ; i++) { swap = a[i]; j = i; - while (Compare::Compare(compobj, swap, a[j-1]) < 0) - { + while (Compare::Compare(compobj, swap, a[j-1]) < 0) { a[j] = a[j-1]; if (!(--j)) break;; } @@ -502,7 +489,7 @@ FastS_SortSpec::sortResults(RankedHit a[], uint32_t n, uint32_t topn) } else if (_method == 1) { std::sort(sortData, sortData + n, StdSortDataCompare(&_binarySortData[0])); } else { - vespalib::Array<uint32_t> radixScratchPad(n, DefaultAlloc::create(0, MMAP_LIMIT)); + vespalib::Array<uint32_t> radixScratchPad(n, Alloc::alloc(0, MMAP_LIMIT)); search::radix_sort(SortDataRadix(&_binarySortData[0]), StdSortDataCompare(&_binarySortData[0]), SortDataEof(), 1, sortData, n, &radixScratchPad[0], 0, 96, topn); } for (uint32_t i(0), m(_sortDataArray.size()); i < m; ++i) { diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.cpp b/searchlib/src/vespa/searchlib/docstore/compacter.cpp index d787f934efc..44032a1a73d 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.cpp +++ b/searchlib/src/vespa/searchlib/docstore/compacter.cpp @@ -9,6 +9,8 @@ LOG_SETUP(".searchlib.docstore.compacter"); namespace search { namespace docstore { +using vespalib::alloc::Alloc; + void Compacter::write(LockGuard guard, uint32_t chunkId, uint32_t lid, const void *buffer, size_t sz) { (void) chunkId; @@ -24,7 +26,7 @@ BucketCompacter::BucketCompacter(size_t maxSignificantBucketBits, const Compress _bucketizer(bucketizer), _writeCount(0), _lock(), - _backingMemory(vespalib::DefaultAlloc::create(0x40000000), &_lock), + _backingMemory(Alloc::alloc(0x40000000), &_lock), _tmpStore(), _lidGuard(ds.getLidReadGuard()), _bucketizerGuard(bucketizer.getGuard()), diff --git a/searchlib/src/vespa/searchlib/docstore/documentstore.h b/searchlib/src/vespa/searchlib/docstore/documentstore.h index 7ca3b98d525..43006b18e90 100644 --- a/searchlib/src/vespa/searchlib/docstore/documentstore.h +++ b/searchlib/src/vespa/searchlib/docstore/documentstore.h @@ -159,6 +159,7 @@ private: class WrapVisitorProgress; class Value { public: + using Alloc = vespalib::alloc::Alloc; typedef std::unique_ptr<Value> UP; Value() : _compressedSize(0), _uncompressedSize(0), _compression(document::CompressionConfig::NONE) { } @@ -173,7 +174,7 @@ private: _compressedSize(rhs._compressedSize), _uncompressedSize(rhs._uncompressedSize), _compression(rhs._compression), - _buf(vespalib::DefaultAlloc::create(rhs.size())) + _buf(Alloc::alloc(rhs.size())) { memcpy(get(), rhs.get(), size()); } @@ -213,7 +214,7 @@ private: size_t _compressedSize; size_t _uncompressedSize; document::CompressionConfig::Type _compression; - vespalib::alloc::Alloc _buf; + Alloc _buf; }; class BackingStore { public: diff --git a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp index 89bfb2bfae4..2f957410bb9 100644 --- a/searchlib/src/vespa/searchlib/docstore/visitcache.cpp +++ b/searchlib/src/vespa/searchlib/docstore/visitcache.cpp @@ -10,7 +10,6 @@ using vespalib::LockGuard; using vespalib::DataBuffer; using vespalib::alloc::Alloc; using vespalib::alloc::MemoryAllocator; -using vespalib::DefaultAlloc; KeySet::KeySet(uint32_t key) : _keys() @@ -31,7 +30,7 @@ KeySet::contains(const KeySet &rhs) const { BlobSet::BlobSet() : _positions(), - _buffer(DefaultAlloc::create(0, 16 * MemoryAllocator::HUGEPAGE_SIZE), 0) + _buffer(Alloc::alloc(0, 16 * MemoryAllocator::HUGEPAGE_SIZE), 0) { } namespace { @@ -92,7 +91,7 @@ BlobSet CompressedBlobSet::getBlobSet() const { // These are frequent lage allocations that are to expensive to mmap. - DataBuffer uncompressed(0, 1, DefaultAlloc::create(0, 16 * MemoryAllocator::HUGEPAGE_SIZE)); + DataBuffer uncompressed(0, 1, Alloc::alloc(0, 16 * MemoryAllocator::HUGEPAGE_SIZE)); if ( ! _positions.empty() ) { document::decompress(_compression, getBufferSize(_positions), ConstBufferRef(_buffer.c_str(), _buffer.size()), uncompressed, false); } diff --git a/searchlib/src/vespa/searchlib/grouping/sketch.h b/searchlib/src/vespa/searchlib/grouping/sketch.h index e336955f337..b97da00f4cf 100644 --- a/searchlib/src/vespa/searchlib/grouping/sketch.h +++ b/searchlib/src/vespa/searchlib/grouping/sketch.h @@ -233,7 +233,7 @@ decompress_buckets_from(char *buffer, uint32_t size) { template <int BucketBits, typename HashT> void NormalSketch<BucketBits, HashT>:: serialize(vespalib::Serializer &os) const { - vespalib::alloc::Alloc backing(vespalib::DefaultAlloc::create(LZ4_compressBound(BUCKET_COUNT))); + vespalib::alloc::Alloc backing(vespalib::alloc::Alloc::alloc(LZ4_compressBound(BUCKET_COUNT))); char * compress_array(static_cast<char *>(backing.get())); uint32_t size = compress_buckets_into(compress_array, backing.size()); os << BUCKET_COUNT << size; diff --git a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp index 1610267c255..6ad7766441a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.cpp @@ -250,7 +250,7 @@ void PredicateBlueprint::fetchPostings(bool) { } PredicateAttribute::MinFeatureHandle mfh = predicate_attribute().getMinFeatureVector(); - vespalib::alloc::Alloc kv(vespalib::DefaultAlloc::create(mfh.second)); + Alloc kv(Alloc::alloc(mfh.second)); _kVBacking.swap(kv); _kV = BitVectorCache::CountVector(static_cast<uint8_t *>(_kVBacking.get()), mfh.second); _index.computeCountVector(_cachedFeatures, _kV); diff --git a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.h index 616460f9e55..140ee17fc62 100644 --- a/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/predicate_blueprint.h @@ -58,6 +58,7 @@ private: using VectorIterator = predicate::SimpleIndex<btree::EntryRef>::VectorIterator; template <typename T> using optional = std::experimental::optional<T>; + using Alloc = vespalib::alloc::Alloc; const PredicateAttribute & predicate_attribute() const { return _attribute; @@ -72,7 +73,7 @@ private: const PredicateAttribute & _attribute; const predicate::PredicateIndex &_index; - vespalib::alloc::Alloc _kVBacking; + Alloc _kVBacking; BitVectorCache::CountVector _kV; BitVectorCache::KeySet _cachedFeatures; diff --git a/searchlib/src/vespa/searchlib/test/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/CMakeLists.txt index 1e0bcb67da6..0632e8a4a91 100644 --- a/searchlib/src/vespa/searchlib/test/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/CMakeLists.txt @@ -8,5 +8,6 @@ vespa_add_library(searchlib_test $<TARGET_OBJECTS:searchlib_test_fakedata> $<TARGET_OBJECTS:searchlib_searchlib_test_diskindex> DEPENDS + searchlib searchlib_searchlib_test_memoryindex ) diff --git a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp index 776924a8c85..edec458c3b9 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domainpart.cpp @@ -21,18 +21,16 @@ using vespalib::getLastErrorString; using vespalib::IllegalHeaderException; using vespalib::LockGuard; using vespalib::nbostream; +using vespalib::alloc::Alloc; using search::common::FileHeaderContext; using std::runtime_error; -namespace search -{ +namespace search { -namespace transactionlog -{ +namespace transactionlog { +namespace { -namespace -{ void handleSync(FastOS_FileInterface &file) __attribute__ ((noinline)); @@ -221,7 +219,7 @@ DomainPart::buildPacketMapping(bool allowTruncate) SerialNum lastSerial(0); int64_t firstPos(currPos); bool full(false); - vespalib::alloc::Alloc buf; + Alloc buf; for(size_t i(0); !full && (currPos < fSize); i++) { Packet::Entry e; if (read(transLog, e, buf, allowTruncate)) { @@ -552,7 +550,7 @@ DomainPart::visit(FastOS_FileInterface &file, SerialNumRange &r, Packet &packet) } if (retval) { Packet newPacket; - vespalib::alloc::Alloc buf; + Alloc buf; for (bool full(false);!full && retval && (r.from() < r.to());) { Packet::Entry e; int64_t fPos = file.GetPosition(); @@ -612,7 +610,7 @@ DomainPart::write(FastOS_FileInterface &file, const Packet::Entry &entry) bool DomainPart::read(FastOS_FileInterface &file, Packet::Entry &entry, - vespalib::alloc::Alloc & buf, + Alloc & buf, bool allowTruncate) { bool retval(true); @@ -625,7 +623,7 @@ DomainPart::read(FastOS_FileInterface &file, his >> version >> len; if ((retval = (rlen == sizeof(tmp)))) { if ( ! (retval = (version == ccitt_crc32) || version == xxh64)) { - vespalib::string msg(make_string("Version mismatch. Expected 'ccitt_crc32=1' or 'xxh64=2'," + string msg(make_string("Version mismatch. Expected 'ccitt_crc32=1' or 'xxh64=2'," " got %d from '%s' at position %ld", version, file.GetFileName(), lastKnownGoodPos)); if ((version == 0) && (len == 0) && tailOfFileIsZero(file, lastKnownGoodPos)) { @@ -636,7 +634,7 @@ DomainPart::read(FastOS_FileInterface &file, } } if (len > buf.size()) { - vespalib::DefaultAlloc::create(len).swap(buf); + Alloc::alloc(len).swap(buf); } rlen = file.Read(buf.get(), len); retval = rlen == len; diff --git a/searchlib/src/vespa/searchlib/uca/CMakeLists.txt b/searchlib/src/vespa/searchlib/uca/CMakeLists.txt index 7ff31f7c7a0..f8955bef1e0 100644 --- a/searchlib/src/vespa/searchlib/uca/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/uca/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_library(searchlib_searchlib_uca ucafunctionnode.cpp INSTALL lib64 DEPENDS + searchlib icui18n icuuc ) diff --git a/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp b/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp index 57a18fc5573..34bf061bb32 100644 --- a/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp +++ b/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp @@ -21,7 +21,7 @@ public: void MemoryDataStoreTest::testMemoryDataStore() { - MemoryDataStore s(DefaultAlloc::create(256)); + MemoryDataStore s(alloc::Alloc::alloc(256)); std::vector<MemoryDataStore::Reference> v; v.push_back(s.push_back("mumbo", 5)); for (size_t i(0); i < 50; i++) { diff --git a/staging_vespalib/src/tests/state_server/state_server_test.cpp b/staging_vespalib/src/tests/state_server/state_server_test.cpp index b1593d2c70e..a5df5778368 100644 --- a/staging_vespalib/src/tests/state_server/state_server_test.cpp +++ b/staging_vespalib/src/tests/state_server/state_server_test.cpp @@ -322,13 +322,20 @@ struct EchoConsumer : MetricsProducer { } }; -TEST_FFFF("require that empty metrics consumer defaults to 'statereporter'", +TEST_FFFF("require that empty v1 metrics consumer defaults to 'statereporter'", SimpleHealthProducer(), EchoConsumer(), SimpleComponentConfigProducer(), StateApi(f1, f2, f3)) { std::map<vespalib::string,vespalib::string> my_params; EXPECT_EQUAL("{\"status\":{\"code\":\"up\"},\"metrics\":[\"statereporter\"]}", f4.get(host_tag, metrics_path, empty_params)); - EXPECT_EQUAL("[\"statereporter\"]", f4.get(host_tag, total_metrics_path, empty_params)); +} + +TEST_FFFF("require that empty total metrics consumer defaults to the empty string", + SimpleHealthProducer(), EchoConsumer(), SimpleComponentConfigProducer(), + StateApi(f1, f2, f3)) +{ + std::map<vespalib::string,vespalib::string> my_params; + EXPECT_EQUAL("[\"\"]", f4.get(host_tag, total_metrics_path, empty_params)); } TEST_FFFF("require that metrics consumer is passed correctly", diff --git a/staging_vespalib/src/vespa/vespalib/data/databuffer.h b/staging_vespalib/src/vespa/vespalib/data/databuffer.h index 53e9e54d0f5..a9ed53e2f84 100644 --- a/staging_vespalib/src/vespa/vespalib/data/databuffer.h +++ b/staging_vespalib/src/vespa/vespalib/data/databuffer.h @@ -33,7 +33,7 @@ namespace vespalib { class DataBuffer { private: - using Alloc = vespalib::alloc::Alloc; + using Alloc = alloc::Alloc; size_t _alignment; char *_externalBuf; char *_bufstart; @@ -53,7 +53,7 @@ public: * @param len the initial size of the buffer. * @param alignment required memory alignment for data start **/ - DataBuffer(size_t len = 1024, size_t alignment = 1, const Alloc & initial = vespalib::DefaultAlloc::create(0)); + DataBuffer(size_t len = 1024, size_t alignment = 1, const Alloc & initial = Alloc::alloc(0)); /** * Construct a databuffer using externally allocated memory. Note @@ -70,7 +70,7 @@ public: _bufend(buf + len), _datapt(_bufstart), _freept(_bufstart), - _buffer(vespalib::DefaultAlloc::create(0)) + _buffer(Alloc::alloc(0)) { } DataBuffer(const char *buf, size_t len) : @@ -80,7 +80,7 @@ public: _bufend(_bufstart + len), _datapt(_bufstart), _freept(_bufend), - _buffer(vespalib::DefaultAlloc::create(0)) + _buffer(Alloc::alloc(0)) { } /** diff --git a/staging_vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt index 032528482e6..6362cdf6f87 100644 --- a/staging_vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/hwaccelrated/CMakeLists.txt @@ -11,4 +11,4 @@ vespa_add_library(staging_vespalib_vespalib_hwaccelrated OBJECT ) set_source_files_properties(avx.cpp PROPERTIES COMPILE_FLAGS -march=sandybridge) set_source_files_properties(avx2.cpp PROPERTIES COMPILE_FLAGS -march=haswell) -set_source_files_properties(avx512.cpp PROPERTIES COMPILE_FLAGS -march=skylake-avx512) +vespa_workaround_set_gcc_march_skylake_avx512_if_supported(avx512.cpp) diff --git a/staging_vespalib/src/vespa/vespalib/net/state_api.cpp b/staging_vespalib/src/vespa/vespalib/net/state_api.cpp index 1885de0f99b..924aec54cf2 100644 --- a/staging_vespalib/src/vespa/vespalib/net/state_api.cpp +++ b/staging_vespalib/src/vespa/vespalib/net/state_api.cpp @@ -53,12 +53,12 @@ void build_health_status(JSONStringer &json, const HealthProducer &healthProduce json.endObject(); } -vespalib::string get_consumer(const std::map<vespalib::string,vespalib::string> ¶ms) { +vespalib::string get_consumer(const std::map<vespalib::string,vespalib::string> ¶ms, + vespalib::stringref default_consumer) +{ auto consumer_lookup = params.find("consumer"); if (consumer_lookup == params.end()) { - // Using a 'statereporter' consumer removes many uninteresting per-thread - // metrics but retains their aggregates. - return "statereporter"; + return default_consumer; } return consumer_lookup->second; } @@ -144,11 +144,13 @@ StateApi::get(const vespalib::string &host, } else if (path == "/state/v1/health") { return respond_health(_healthProducer); } else if (path == "/state/v1/metrics") { - return respond_metrics(get_consumer(params), _healthProducer, _metricsProducer); + // Using a 'statereporter' consumer by default removes many uninteresting per-thread + // metrics but retains their aggregates. + return respond_metrics(get_consumer(params, "statereporter"), _healthProducer, _metricsProducer); } else if (path == "/state/v1/config") { return respond_config(_componentConfigProducer); } else if (path == "/metrics/total") { - return _metricsProducer.getTotalMetrics(get_consumer(params)); + return _metricsProducer.getTotalMetrics(get_consumer(params, "")); } else { return _handler_repo.get(host, path, params); } diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp index b17accddc5d..abd5e7a2b5b 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp @@ -5,7 +5,7 @@ using namespace vespalib; GrowableByteBuffer::GrowableByteBuffer(uint32_t initialLen) : - _buffer(DefaultAlloc::create(initialLen)), + _buffer(Alloc::alloc(initialLen)), _position(0) { } @@ -16,7 +16,7 @@ GrowableByteBuffer::allocate(uint32_t len) size_t need(_position + len); if (need > _buffer.size()) { uint32_t newSize = vespalib::roundUp2inN(need); - Alloc newBuf(DefaultAlloc::create(newSize)); + Alloc newBuf(Alloc::alloc(newSize)); memcpy(newBuf.get(), _buffer.get(), _position); _buffer.swap(newBuf); } diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index bada6980b89..e0c0ab78824 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -63,6 +63,7 @@ vespa_define_module( src/tests/frameworkimpl/memory src/tests/frameworkimpl/status src/tests/persistence + src/tests/persistence/common src/tests/persistence/filestorage src/tests/storageserver src/tests/storageutil diff --git a/storage/src/tests/CMakeLists.txt b/storage/src/tests/CMakeLists.txt index 03fbefed448..fb5bc146983 100644 --- a/storage/src/tests/CMakeLists.txt +++ b/storage/src/tests/CMakeLists.txt @@ -15,8 +15,5 @@ vespa_add_executable(storage_testrunner_app TEST storage_testfilestorage storage_testmemory storage_teststatus - storage - AFTER - storage_storageconfig ) vespa_add_test(NAME storage_testrunner_app COMMAND storage_testrunner_app) diff --git a/storage/src/tests/bucketdb/CMakeLists.txt b/storage/src/tests/bucketdb/CMakeLists.txt index 2ba195e70e0..767c7ccf6fc 100644 --- a/storage/src/tests/bucketdb/CMakeLists.txt +++ b/storage/src/tests/bucketdb/CMakeLists.txt @@ -8,7 +8,7 @@ vespa_add_library(storage_testbucketdb TEST lockablemaptest.cpp bucketinfotest.cpp distribution_hash_normalizer_test.cpp - AFTER - storage_storageconfig - storage_bucketdb + DEPENDS + storage + storage_testcommon ) diff --git a/storage/src/tests/bucketdb/bucketinfotest.cpp b/storage/src/tests/bucketdb/bucketinfotest.cpp index eef4c6d7739..9ee5895a41f 100644 --- a/storage/src/tests/bucketdb/bucketinfotest.cpp +++ b/storage/src/tests/bucketdb/bucketinfotest.cpp @@ -8,7 +8,7 @@ #include <map> #include <vector> #include <vespa/vespalib/text/stringtokenizer.h> -#include <vespa/storage/distributor/bucketdb/bucketinfo.h> +#include <vespa/storage/bucketdb/bucketinfo.h> namespace storage { diff --git a/storage/src/tests/bucketmover/CMakeLists.txt b/storage/src/tests/bucketmover/CMakeLists.txt index e2929187f62..9181edcc644 100644 --- a/storage/src/tests/bucketmover/CMakeLists.txt +++ b/storage/src/tests/bucketmover/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(storage_testbucketmover TEST SOURCES bucketmovertest.cpp htmltabletest.cpp - AFTER - storage_storageconfig + DEPENDS + storage + storage_testcommon ) diff --git a/storage/src/tests/common/CMakeLists.txt b/storage/src/tests/common/CMakeLists.txt index 737815d85c2..13ac34c2726 100644 --- a/storage/src/tests/common/CMakeLists.txt +++ b/storage/src/tests/common/CMakeLists.txt @@ -6,7 +6,6 @@ vespa_add_library(storage_testcommon TEST metricstest.cpp storagelinktest.cpp teststorageapp.cpp - AFTER - storage_storageconfig - storage_bucketdb + DEPENDS + storage ) diff --git a/storage/src/tests/common/hostreporter/CMakeLists.txt b/storage/src/tests/common/hostreporter/CMakeLists.txt index 9e3fabdcb29..8a849cf884f 100644 --- a/storage/src/tests/common/hostreporter/CMakeLists.txt +++ b/storage/src/tests/common/hostreporter/CMakeLists.txt @@ -8,6 +8,6 @@ vespa_add_library(storage_testhostreporter TEST diskreportertest.cpp util.cpp hostinfotest.cpp - AFTER - storage_storageconfig + DEPENDS + storage ) diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h index e7da9178743..14f9aad3ab0 100644 --- a/storage/src/tests/common/teststorageapp.h +++ b/storage/src/tests/common/teststorageapp.h @@ -19,7 +19,6 @@ #include <vespa/document/base/testdocman.h> #include <vespa/persistence/spi/persistenceprovider.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/common/doneinitializehandler.h> #include <vespa/storage/common/nodestateupdater.h> @@ -100,7 +99,7 @@ private: // Storage server interface implementation (until we can remove it) virtual api::Timestamp getUniqueTimestamp() { assert(0); throw; } virtual StorBucketDatabase& getStorageBucketDatabase() { assert(0); throw; } - virtual distributor::BucketDatabase& getBucketDatabase() { assert(0); throw; } + virtual BucketDatabase& getBucketDatabase() { assert(0); throw; } virtual uint16_t getDiskCount() const { assert(0); throw; } }; @@ -151,7 +150,7 @@ public: DistributorComponentRegisterImpl& getComponentRegister() { return _compReg; } - virtual distributor::BucketDatabase& getBucketDatabase() + virtual BucketDatabase& getBucketDatabase() { return _compReg.getBucketDatabase(); } virtual api::Timestamp getUniqueTimestamp(); diff --git a/storage/src/tests/distributor/CMakeLists.txt b/storage/src/tests/distributor/CMakeLists.txt index 29bb8cb67c1..30751a01a30 100644 --- a/storage/src/tests/distributor/CMakeLists.txt +++ b/storage/src/tests/distributor/CMakeLists.txt @@ -38,6 +38,8 @@ vespa_add_library(storage_testdistributor TEST bucketgctimecalculatortest.cpp nodemaintenancestatstrackertest.cpp distributor_host_info_reporter_test.cpp - AFTER - storage_storageconfig + DEPENDS + storage_distributor + storage_testcommon + storage_testhostreporter ) diff --git a/storage/src/tests/distributor/bucketdatabasetest.h b/storage/src/tests/distributor/bucketdatabasetest.h index 1eb8bf86add..f9429de8b1f 100644 --- a/storage/src/tests/distributor/bucketdatabasetest.h +++ b/storage/src/tests/distributor/bucketdatabasetest.h @@ -2,10 +2,11 @@ #pragma once #include <vespa/fastos/fastos.h> +#include <vespa/document/bucket/bucketid.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/vespalib/util/document_runnable.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <cppunit/extensions/HelperMacros.h> -#include <vespa/storage/distributor/bucketdb/judybucketdatabase.h> #include <vespa/storage/storageutil/utils.h> #define SETUP_DATABASE_TESTS() \ diff --git a/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp index 6aa9ef3a844..135d5c65110 100644 --- a/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp +++ b/storage/src/tests/distributor/bucketdbmetricupdatertest.cpp @@ -3,8 +3,8 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <string> #include <sstream> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h> -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storage/distributor/idealstatemetricsset.h> #include <vespa/storage/config/config-stor-distributormanager.h> diff --git a/storage/src/tests/distributor/distributortest.cpp b/storage/src/tests/distributor/distributortest.cpp index b51c8dd3873..69591a38c45 100644 --- a/storage/src/tests/distributor/distributortest.cpp +++ b/storage/src/tests/distributor/distributortest.cpp @@ -5,7 +5,6 @@ #include <memory> #include <boost/assign/std/vector.hpp> // for 'operator+=()' #include <vespa/vdstestlib/cppunit/macros.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/idealstatemetricsset.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/bucketsplitting.h> diff --git a/storage/src/tests/distributor/maintenanceschedulertest.cpp b/storage/src/tests/distributor/maintenanceschedulertest.cpp index 4316bfd137c..dfc81c1110c 100644 --- a/storage/src/tests/distributor/maintenanceschedulertest.cpp +++ b/storage/src/tests/distributor/maintenanceschedulertest.cpp @@ -6,7 +6,7 @@ #include <memory> #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> #include <vespa/storage/distributor/maintenance/maintenancescheduler.h> -#include <vespa/storage/distributor/bucketdb/mapbucketdatabase.h> +#include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <tests/distributor/maintenancemocks.h> namespace storage { diff --git a/storage/src/tests/distributor/mapbucketdatabasetest.cpp b/storage/src/tests/distributor/mapbucketdatabasetest.cpp index ab8e5add65f..fbc13d5a8a2 100644 --- a/storage/src/tests/distributor/mapbucketdatabasetest.cpp +++ b/storage/src/tests/distributor/mapbucketdatabasetest.cpp @@ -3,7 +3,7 @@ #include <vespa/vespalib/util/document_runnable.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <cppunit/extensions/HelperMacros.h> -#include <vespa/storage/distributor/bucketdb/mapbucketdatabase.h> +#include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <vespa/storage/storageutil/utils.h> #include <tests/distributor/bucketdatabasetest.h> diff --git a/storage/src/tests/distributor/simplemaintenancescannertest.cpp b/storage/src/tests/distributor/simplemaintenancescannertest.cpp index 512a10bbd9a..3486f41149b 100644 --- a/storage/src/tests/distributor/simplemaintenancescannertest.cpp +++ b/storage/src/tests/distributor/simplemaintenancescannertest.cpp @@ -4,7 +4,7 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/distributor/maintenance/simplemaintenancescanner.h> #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> -#include <vespa/storage/distributor/bucketdb/mapbucketdatabase.h> +#include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <tests/distributor/maintenancemocks.h> #include <string> diff --git a/storage/src/tests/frameworkimpl/memory/CMakeLists.txt b/storage/src/tests/frameworkimpl/memory/CMakeLists.txt index 4d89a4aa548..77e0b9a2a9e 100644 --- a/storage/src/tests/frameworkimpl/memory/CMakeLists.txt +++ b/storage/src/tests/frameworkimpl/memory/CMakeLists.txt @@ -2,6 +2,7 @@ vespa_add_library(storage_testmemory TEST SOURCES memorystatusviewertest.cpp - AFTER - storage_storageconfig + DEPENDS + storage + storage_testcommon ) diff --git a/storage/src/tests/frameworkimpl/status/CMakeLists.txt b/storage/src/tests/frameworkimpl/status/CMakeLists.txt index 95c4d3a532e..2de52df1f7f 100644 --- a/storage/src/tests/frameworkimpl/status/CMakeLists.txt +++ b/storage/src/tests/frameworkimpl/status/CMakeLists.txt @@ -2,6 +2,7 @@ vespa_add_library(storage_teststatus TEST SOURCES statustest.cpp - AFTER - storage_storageconfig + DEPENDS + storage + storage_testcommon ) diff --git a/storage/src/tests/persistence/CMakeLists.txt b/storage/src/tests/persistence/CMakeLists.txt index db0ac64c183..8863b3d0467 100644 --- a/storage/src/tests/persistence/CMakeLists.txt +++ b/storage/src/tests/persistence/CMakeLists.txt @@ -5,7 +5,6 @@ vespa_add_library(storage_testpersistence TEST persistencetestutils.cpp splitbitdetectortest.cpp legacyoperationhandlertest.cpp - persistenceproviderwrapper.cpp diskmoveoperationhandlertest.cpp providershutdownwrappertest.cpp mergehandlertest.cpp @@ -13,6 +12,8 @@ vespa_add_library(storage_testpersistence TEST bucketownershipnotifiertest.cpp persistencequeuetest.cpp testandsettest.cpp - AFTER - storage_storageconfig + DEPENDS + storage + storage_testdistributor + storage_testpersistence_common ) diff --git a/storage/src/tests/persistence/common/CMakeLists.txt b/storage/src/tests/persistence/common/CMakeLists.txt new file mode 100644 index 00000000000..561f516b8d1 --- /dev/null +++ b/storage/src/tests/persistence/common/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(storage_testpersistence_common TEST + SOURCES + persistenceproviderwrapper.cpp + filestortestfixture.cpp + DEPENDS + persistence + storage_testcommon +) diff --git a/storage/src/tests/persistence/filestorage/filestortestfixture.cpp b/storage/src/tests/persistence/common/filestortestfixture.cpp index 69b109b5cfc..8f7a95f53c8 100644 --- a/storage/src/tests/persistence/filestorage/filestortestfixture.cpp +++ b/storage/src/tests/persistence/common/filestortestfixture.cpp @@ -5,7 +5,7 @@ #include <vespa/storage/persistence/messages.h> #include <vespa/storage/persistence/filestorage/filestormanager.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> -#include <tests/persistence/filestorage/filestortestfixture.h> +#include <tests/persistence/common/filestortestfixture.h> namespace storage { diff --git a/storage/src/tests/persistence/filestorage/filestortestfixture.h b/storage/src/tests/persistence/common/filestortestfixture.h index 4f1de549f47..4f1de549f47 100644 --- a/storage/src/tests/persistence/filestorage/filestortestfixture.h +++ b/storage/src/tests/persistence/common/filestortestfixture.h diff --git a/storage/src/tests/persistence/persistenceproviderwrapper.cpp b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp index 4a09235ddce..9ec66590b24 100644 --- a/storage/src/tests/persistence/persistenceproviderwrapper.cpp +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.cpp @@ -2,7 +2,7 @@ #include <vespa/fastos/fastos.h> #include <iostream> #include <sstream> -#include <tests/persistence/persistenceproviderwrapper.h> +#include <tests/persistence/common/persistenceproviderwrapper.h> #define LOG_SPI(ops) \ { \ diff --git a/storage/src/tests/persistence/persistenceproviderwrapper.h b/storage/src/tests/persistence/common/persistenceproviderwrapper.h index b115eb7ef3d..b115eb7ef3d 100644 --- a/storage/src/tests/persistence/persistenceproviderwrapper.h +++ b/storage/src/tests/persistence/common/persistenceproviderwrapper.h diff --git a/storage/src/tests/persistence/filestorage/CMakeLists.txt b/storage/src/tests/persistence/filestorage/CMakeLists.txt index 2ccebdd8a96..f253d42e460 100644 --- a/storage/src/tests/persistence/filestorage/CMakeLists.txt +++ b/storage/src/tests/persistence/filestorage/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_add_library(storage_testfilestorage TEST SOURCES filestormanagertest.cpp operationabortingtest.cpp - filestortestfixture.cpp mergeblockingtest.cpp sanitycheckeddeletetest.cpp deactivatebucketstest.cpp @@ -11,7 +10,8 @@ vespa_add_library(storage_testfilestorage TEST filestormodifiedbucketstest.cpp deletebuckettest.cpp singlebucketjointest.cpp - AFTER - storage_bucketdb - storage_storageconfig + DEPENDS + storage + storageapi + storage_testpersistence_common ) diff --git a/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp b/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp index 6de67a3fec0..c4e3ef97eb2 100644 --- a/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp +++ b/storage/src/tests/persistence/filestorage/deactivatebucketstest.cpp @@ -4,9 +4,9 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> -#include <tests/persistence/persistenceproviderwrapper.h> +#include <tests/persistence/common/persistenceproviderwrapper.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> -#include <tests/persistence/filestorage/filestortestfixture.h> +#include <tests/persistence/common/filestortestfixture.h> namespace storage { diff --git a/storage/src/tests/persistence/filestorage/deletebuckettest.cpp b/storage/src/tests/persistence/filestorage/deletebuckettest.cpp index 08ca9bc68fa..d415db1b45e 100644 --- a/storage/src/tests/persistence/filestorage/deletebuckettest.cpp +++ b/storage/src/tests/persistence/filestorage/deletebuckettest.cpp @@ -4,9 +4,9 @@ #include <vespa/log/log.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/bucket.h> -#include <tests/persistence/persistenceproviderwrapper.h> +#include <tests/persistence/common/persistenceproviderwrapper.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> -#include <tests/persistence/filestorage/filestortestfixture.h> +#include <tests/persistence/common/filestortestfixture.h> LOG_SETUP(".deletebuckettest"); diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 0ffbe9fa440..251b2474154 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -2264,7 +2264,7 @@ FileStorManagerTest::testDeleteBucketRejectOutdatedBucketInfo() { std::shared_ptr<api::DeleteBucketCommand> cmd( new api::DeleteBucketCommand(bid)); - cmd->setBucketInfo(BucketInfo(0xf000baaa, 1, 123, 1, 456)); + cmd->setBucketInfo(api::BucketInfo(0xf000baaa, 1, 123, 1, 456)); cmd->setAddress(address); top.sendDown(cmd); top.waitForMessages(1, _waitTime); @@ -2777,7 +2777,7 @@ FileStorManagerTest::testGetIter() _node->getTestDocMan().createRandomDocumentAtLocation( 4000, i, 400, 400))); } - BucketInfo bucketInfo; + api::BucketInfo bucketInfo; // Putting all docs to have something to visit for (uint32_t i=0; i<docs.size(); ++i) { std::shared_ptr<api::PutCommand> cmd( diff --git a/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp index 19b84ef475b..8d6a8fb8e03 100644 --- a/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormodifiedbucketstest.cpp @@ -4,9 +4,9 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storage/persistence/filestorage/modifiedbucketchecker.h> -#include <tests/persistence/persistenceproviderwrapper.h> +#include <tests/persistence/common/persistenceproviderwrapper.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> -#include <tests/persistence/filestorage/filestortestfixture.h> +#include <tests/persistence/common/filestortestfixture.h> namespace storage { diff --git a/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp b/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp index ff9ec063555..fd106fb3117 100644 --- a/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp +++ b/storage/src/tests/persistence/filestorage/mergeblockingtest.cpp @@ -4,9 +4,9 @@ #include <vector> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/persistence/messages.h> -#include <tests/persistence/persistenceproviderwrapper.h> +#include <tests/persistence/common/persistenceproviderwrapper.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> -#include <tests/persistence/filestorage/filestortestfixture.h> +#include <tests/persistence/common/filestortestfixture.h> namespace storage { diff --git a/storage/src/tests/persistence/filestorage/operationabortingtest.cpp b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp index 0d6583cacdb..e0d2bae9a44 100644 --- a/storage/src/tests/persistence/filestorage/operationabortingtest.cpp +++ b/storage/src/tests/persistence/filestorage/operationabortingtest.cpp @@ -4,9 +4,9 @@ #include <vector> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storage/persistence/messages.h> -#include <tests/persistence/persistenceproviderwrapper.h> +#include <tests/persistence/common/persistenceproviderwrapper.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> -#include <tests/persistence/filestorage/filestortestfixture.h> +#include <tests/persistence/common/filestortestfixture.h> #include <vespa/vespalib/util/barrier.h> #include <vespa/vespalib/util/thread.h> diff --git a/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp b/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp index 9b492a3aaa6..19841e98e85 100644 --- a/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp +++ b/storage/src/tests/persistence/filestorage/sanitycheckeddeletetest.cpp @@ -3,9 +3,9 @@ #include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/bucket.h> -#include <tests/persistence/persistenceproviderwrapper.h> +#include <tests/persistence/common/persistenceproviderwrapper.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> -#include <tests/persistence/filestorage/filestortestfixture.h> +#include <tests/persistence/common/filestortestfixture.h> namespace storage { diff --git a/storage/src/tests/persistence/filestorage/singlebucketjointest.cpp b/storage/src/tests/persistence/filestorage/singlebucketjointest.cpp index 480652207d3..c754ca353af 100644 --- a/storage/src/tests/persistence/filestorage/singlebucketjointest.cpp +++ b/storage/src/tests/persistence/filestorage/singlebucketjointest.cpp @@ -5,9 +5,9 @@ #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/bucketsplitting.h> -#include <tests/persistence/persistenceproviderwrapper.h> +#include <tests/persistence/common/persistenceproviderwrapper.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> -#include <tests/persistence/filestorage/filestortestfixture.h> +#include <tests/persistence/common/filestortestfixture.h> LOG_SETUP(".singlebucketjointest"); diff --git a/storage/src/tests/persistence/mergehandlertest.cpp b/storage/src/tests/persistence/mergehandlertest.cpp index b0ec11a3e46..48d2ef2dc61 100644 --- a/storage/src/tests/persistence/mergehandlertest.cpp +++ b/storage/src/tests/persistence/mergehandlertest.cpp @@ -7,7 +7,7 @@ #include <vespa/storageapi/message/bucket.h> #include <vespa/log/log.h> #include <tests/persistence/persistencetestutils.h> -#include <tests/persistence/persistenceproviderwrapper.h> +#include <tests/persistence/common/persistenceproviderwrapper.h> #include <tests/distributor/messagesenderstub.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> #include <cmath> diff --git a/storage/src/tests/persistence/persistencequeuetest.cpp b/storage/src/tests/persistence/persistencequeuetest.cpp index 06daf2a975c..3a262c01d54 100644 --- a/storage/src/tests/persistence/persistencequeuetest.cpp +++ b/storage/src/tests/persistence/persistencequeuetest.cpp @@ -4,9 +4,9 @@ #include <vespa/log/log.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <vespa/storageapi/message/bucket.h> -#include <tests/persistence/persistenceproviderwrapper.h> +#include <tests/persistence/common/persistenceproviderwrapper.h> #include <vespa/persistence/dummyimpl/dummypersistence.h> -#include <tests/persistence/filestorage/filestortestfixture.h> +#include <tests/persistence/common/filestortestfixture.h> #include <tests/persistence/filestorage/forwardingmessagesender.h> LOG_SETUP(".persistencequeuetest"); diff --git a/storage/src/tests/persistence/providershutdownwrappertest.cpp b/storage/src/tests/persistence/providershutdownwrappertest.cpp index 0731dcb155a..1d2bf08e0c4 100644 --- a/storage/src/tests/persistence/providershutdownwrappertest.cpp +++ b/storage/src/tests/persistence/providershutdownwrappertest.cpp @@ -3,7 +3,7 @@ #include <vespa/fastos/fastos.h> #include <vespa/vdstestlib/cppunit/macros.h> #include <tests/persistence/persistencetestutils.h> -#include <tests/persistence/persistenceproviderwrapper.h> +#include <tests/persistence/common/persistenceproviderwrapper.h> #include <vespa/storage/persistence/providershutdownwrapper.h> namespace storage { diff --git a/storage/src/tests/storageserver/CMakeLists.txt b/storage/src/tests/storageserver/CMakeLists.txt index dacd37d8ea7..e415c76ae83 100644 --- a/storage/src/tests/storageserver/CMakeLists.txt +++ b/storage/src/tests/storageserver/CMakeLists.txt @@ -11,7 +11,7 @@ vespa_add_library(storage_teststorageserver TEST priorityconvertertest.cpp statereportertest.cpp changedbucketownershiphandlertest.cpp - AFTER - storage_bucketdb - storage_storageconfig + DEPENDS + storage_storageserver + storage_testcommon ) diff --git a/storage/src/tests/storageutil/CMakeLists.txt b/storage/src/tests/storageutil/CMakeLists.txt index e49361285d8..5fc955a0b1e 100644 --- a/storage/src/tests/storageutil/CMakeLists.txt +++ b/storage/src/tests/storageutil/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_library(storage_teststorageutil TEST functortest.cpp charttest.cpp palettetest.cpp - AFTER - storage_storageconfig + DEPENDS + storage ) diff --git a/storage/src/tests/visiting/CMakeLists.txt b/storage/src/tests/visiting/CMakeLists.txt index f1c10bde4f0..de73896a203 100644 --- a/storage/src/tests/visiting/CMakeLists.txt +++ b/storage/src/tests/visiting/CMakeLists.txt @@ -5,7 +5,7 @@ vespa_add_library(storage_testvisiting TEST visitormanagertest.cpp visitortest.cpp memory_bounded_trace_test.cpp - AFTER - storage_storageconfig - storage_visitor + DEPENDS + storage + storage_teststorageserver ) diff --git a/storage/src/vespa/storage/CMakeLists.txt b/storage/src/vespa/storage/CMakeLists.txt index 2adfe73db38..a04ab6e92d6 100644 --- a/storage/src/vespa/storage/CMakeLists.txt +++ b/storage/src/vespa/storage/CMakeLists.txt @@ -5,13 +5,8 @@ vespa_add_library(storage $<TARGET_OBJECTS:storage_common> $<TARGET_OBJECTS:storage_storageconfig> $<TARGET_OBJECTS:storage_hostreporter> - $<TARGET_OBJECTS:storage_distributoroperation> - $<TARGET_OBJECTS:storage_distributoroperationexternal> - $<TARGET_OBJECTS:storage_distributoroperationidealstate> - $<TARGET_OBJECTS:storage_distributormaintenance> $<TARGET_OBJECTS:storage_filestorpersistence> $<TARGET_OBJECTS:storage_spersistence> - $<TARGET_OBJECTS:storage_storageserver> $<TARGET_OBJECTS:storage_storageutil> $<TARGET_OBJECTS:storage_visitor> $<TARGET_OBJECTS:storage_bucketmover> @@ -21,5 +16,4 @@ vespa_add_library(storage $<TARGET_OBJECTS:storage_component> INSTALL lib64 DEPENDS - storage_distributor ) diff --git a/storage/src/vespa/storage/bucketdb/CMakeLists.txt b/storage/src/vespa/storage/bucketdb/CMakeLists.txt index 189f773dcbe..0a971b0d099 100644 --- a/storage/src/vespa/storage/bucketdb/CMakeLists.txt +++ b/storage/src/vespa/storage/bucketdb/CMakeLists.txt @@ -1,8 +1,11 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(storage_bucketdb OBJECT SOURCES + bucketinfo.cpp + bucketcopy.cpp + bucketdatabase.cpp + mapbucketdatabase.cpp storagebucketdbinitializer.cpp - distrbucketdb.cpp storbucketdb.cpp judyarray.cpp bucketmanager.cpp diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketcopy.cpp b/storage/src/vespa/storage/bucketdb/bucketcopy.cpp index 762fa290109..8bd2a4ff08f 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketcopy.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketcopy.cpp @@ -1,11 +1,9 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/bucketdb/bucketcopy.h> +#include <vespa/storage/bucketdb/bucketcopy.h> namespace storage { -namespace distributor { - void BucketCopy::print(std::ostream& out, bool /*verbose*/, const std::string&) const { @@ -20,5 +18,3 @@ BucketCopy::print(std::ostream& out, bool /*verbose*/, const std::string&) const } } - -} diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketcopy.h b/storage/src/vespa/storage/bucketdb/bucketcopy.h index 5d7c2067fe1..7230c28eb74 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketcopy.h +++ b/storage/src/vespa/storage/bucketdb/bucketcopy.h @@ -5,8 +5,6 @@ namespace storage { -namespace distributor { - class BucketCopy { private: uint64_t _timestamp; @@ -109,5 +107,3 @@ public: } -} - diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdatabase.cpp b/storage/src/vespa/storage/bucketdb/bucketdatabase.cpp index 902b6d88ab0..4a17c2aab59 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdatabase.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketdatabase.cpp @@ -1,9 +1,8 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> namespace storage { -namespace distributor { namespace { struct GetNextEntryProcessor : public BucketDatabase::EntryProcessor { @@ -52,5 +51,3 @@ BucketDatabase::Entry::toString() const } } - -} diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdatabase.h b/storage/src/vespa/storage/bucketdb/bucketdatabase.h index a85b5aa5edf..d17f4a20176 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdatabase.h +++ b/storage/src/vespa/storage/bucketdb/bucketdatabase.h @@ -5,11 +5,10 @@ #pragma once #include <vespa/vespalib/util/printable.h> -#include <vespa/storage/distributor/bucketdb/bucketinfo.h> +#include <vespa/storage/bucketdb/bucketinfo.h> #include <vespa/document/bucket/bucketid.h> namespace storage { -namespace distributor { class BucketDatabase : public vespalib::Printable { @@ -117,5 +116,4 @@ public: std::ostream& operator<<(std::ostream& o, const BucketDatabase::Entry& e); -} // distributor } // storage diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketinfo.cpp b/storage/src/vespa/storage/bucketdb/bucketinfo.cpp index ccb3702bbbd..a9f6a85892b 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketinfo.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketinfo.cpp @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/bucketdb/bucketinfo.h> +#include <vespa/storage/bucketdb/bucketinfo.h> #include <vespa/storage/storageutil/utils.h> namespace storage { -namespace distributor { - BucketInfo::BucketInfo() : _lastGarbageCollection(0) { @@ -312,5 +310,3 @@ BucketInfo::operator==(const BucketInfo& other) const }; } - -} diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketinfo.h b/storage/src/vespa/storage/bucketdb/bucketinfo.h index 4e450994f8a..1b220282cf0 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketinfo.h +++ b/storage/src/vespa/storage/bucketdb/bucketinfo.h @@ -1,11 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/storage/distributor/bucketdb/bucketcopy.h> +#include <vespa/storage/bucketdb/bucketcopy.h> namespace storage { namespace distributor { + class DistributorTestUtil; +} class BucketInfo { @@ -153,8 +155,7 @@ public: bool operator==(const BucketInfo& other) const; private: - friend class DistrBucketDBTest; - friend class DistributorTestUtil; + friend class distributor::DistributorTestUtil; /** * Returns the bucket copy struct for the given node, null if nonexisting @@ -173,5 +174,3 @@ inline std::ostream& operator<<(std::ostream& out, const BucketInfo& info) } -} - diff --git a/storage/src/vespa/storage/bucketdb/distrbucketdb.cpp b/storage/src/vespa/storage/bucketdb/distrbucketdb.cpp deleted file mode 100644 index 6bca384a076..00000000000 --- a/storage/src/vespa/storage/bucketdb/distrbucketdb.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> -#include <vespa/log/log.h> -#include <vespa/storage/storageutil/utils.h> - -LOG_SETUP(".distributor.bucketdb"); - -namespace storage { -namespace bucketdb { - -void -DistrBucketDatabase::insert(const document::BucketId& bucket, - const distributor::BucketInfo& entry, - const char* clientId) -{ - bool preExisted; -#ifdef USE_JUDY - return LockableMap<JudyMultiMap<distributor::BucketInfo> >::insert( - bucket.toKey(), entry, clientId, preExisted); -#else - return LockableMap<StdMapWrapper<document::BucketId::Type, - distributor::BucketInfo> >::insert( - bucket.toKey(), entry, clientId, preExisted); -#endif -} - -DistrBucketDatabase::WrappedEntry -DistrBucketDatabase::get(const document::BucketId& bucket, const char* clientId, - bool createIfNonExisting) -{ -#ifdef USE_JUDY - return LockableMap<JudyMultiMap<distributor::BucketInfo> >::get( - bucket.stripUnused().toKey(), clientId, createIfNonExisting); -#else - return LockableMap<StdMapWrapper<document::BucketId::Type, - distributor::BucketInfo> >::get( - bucket.stripUnused().toKey(), clientId, createIfNonExisting); -#endif -} - -} // storage - -} diff --git a/storage/src/vespa/storage/bucketdb/distrbucketdb.h b/storage/src/vespa/storage/bucketdb/distrbucketdb.h deleted file mode 100644 index 57bdfede403..00000000000 --- a/storage/src/vespa/storage/bucketdb/distrbucketdb.h +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/storage/bucketdb/judymultimap.h> -#include <vespa/storage/bucketdb/lockablemap.h> -#include <vespa/storage/bucketdb/stdmapwrapper.h> -#include <deque> -#include <vespa/vespalib/util/printable.h> -#include <inttypes.h> -#include <map> -#include <stdexcept> -#include <vector> -#include <vespa/vespalib/util/sync.h> -#include <vespa/metrics/valuemetric.h> -#include <vespa/storage/distributor/bucketdb/bucketinfo.h> - -#if __WORDSIZE == 64 - #define USE_JUDY -#endif - -//#undef USE_JUDY - -namespace storage { - -namespace bucketdb { - -class DistrBucketDatabase -#ifdef USE_JUDY - : public LockableMap<JudyMultiMap<distributor::BucketInfo> > -#else - : public LockableMap<StdMapWrapper<document::BucketId::Type, - distributor::BucketInfo> > -#endif -{ -public: - DistrBucketDatabase() {}; - - typedef distributor::BucketInfo Entry; - - void insert(const document::BucketId&, - const distributor::BucketInfo&, - const char* clientId); - - WrappedEntry get(const document::BucketId& bucket, - const char* clientId, - bool createIfNonExisting = false); -}; - -} - -} - - diff --git a/storage/src/vespa/storage/distributor/bucketdb/mapbucketdatabase.cpp b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp index 0abf24e5821..243deac5034 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/mapbucketdatabase.cpp +++ b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.cpp @@ -1,12 +1,10 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/bucketdb/mapbucketdatabase.h> +#include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <vespa/storage/common/bucketoperationlogger.h> namespace storage { -namespace distributor { - MapBucketDatabase::MapBucketDatabase() { // Allocate the root element. @@ -511,5 +509,4 @@ MapBucketDatabase::print(std::ostream& out, bool verbose, } } -} // distributor } // storage diff --git a/storage/src/vespa/storage/distributor/bucketdb/mapbucketdatabase.h b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h index 9574d4fe114..b1d571824c2 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/mapbucketdatabase.h +++ b/storage/src/vespa/storage/bucketdb/mapbucketdatabase.h @@ -1,13 +1,11 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <map> namespace storage { -namespace distributor { - class MapBucketDatabase : public BucketDatabase { public: @@ -107,5 +105,3 @@ private: } -} - diff --git a/storage/src/vespa/storage/common/bucketoperationlogger.cpp b/storage/src/vespa/storage/common/bucketoperationlogger.cpp index 7ec6e2df599..ef4ab1ac8ce 100644 --- a/storage/src/vespa/storage/common/bucketoperationlogger.cpp +++ b/storage/src/vespa/storage/common/bucketoperationlogger.cpp @@ -2,7 +2,7 @@ #include <vespa/fastos/fastos.h> #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/storage/bucketdb/storbucketdb.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> +#include <vespa/storage/bucketdb/bucketcopy.h> #include <vespa/log/log.h> #include <vespa/storageapi/buckets/bucketinfo.h> #include <vespa/storageframework/defaultimplementation/clock/realclock.h> @@ -228,12 +228,12 @@ void logBucketDbErase(uint64_t key, const TypeTag<bucketdb::StorageBucketInfo>&) void checkAllConsistentNodesImpliesTrusted( const document::BucketId& bucket, - const distributor::BucketInfo& entry) + const BucketInfo& entry) { // If all copies are consistent, they should also be trusted if (entry.validAndConsistent() && entry.getNodeCount() > 1) { for (std::size_t i = 0; i < entry.getNodeCount(); ++i) { - const distributor::BucketCopy& copy = entry.getNodeRef(i); + const BucketCopy& copy = entry.getNodeRef(i); if (copy.trusted() == false) { LOG(warning, "Bucket DB entry %s for %s is consistent, but " "contains non-trusted copy %s", entry.toString().c_str(), @@ -245,7 +245,7 @@ checkAllConsistentNodesImpliesTrusted( } std::size_t -firstTrustedNode(const distributor::BucketInfo& entry) +firstTrustedNode(const BucketInfo& entry) { for (std::size_t i = 0; i < entry.getNodeCount(); ++i) { const distributor::BucketCopy& copy = entry.getNodeRef(i); @@ -259,7 +259,7 @@ firstTrustedNode(const distributor::BucketInfo& entry) void checkNotInSyncImpliesNotTrusted( const document::BucketId& bucket, - const distributor::BucketInfo& entry) + const BucketInfo& entry) { // If there are copies out of sync, different copies should not // be set to trusted @@ -267,12 +267,12 @@ checkNotInSyncImpliesNotTrusted( if (trustedNode != std::numeric_limits<std::size_t>::max()) { // Ensure all other trusted copies match the metadata of the // first trusted bucket - const distributor::BucketCopy& trustedCopy = entry.getNodeRef(trustedNode); + const BucketCopy& trustedCopy = entry.getNodeRef(trustedNode); for (std::size_t i = 0; i < entry.getNodeCount(); ++i) { if (i == trustedNode) { continue; } - const distributor::BucketCopy& copy = entry.getNodeRef(i); + const BucketCopy& copy = entry.getNodeRef(i); const api::BucketInfo& copyInfo = copy.getBucketInfo(); const api::BucketInfo& trustedInfo = trustedCopy.getBucketInfo(); if (copy.trusted() @@ -291,10 +291,10 @@ checkNotInSyncImpliesNotTrusted( void checkInvalidImpliesNotTrusted( const document::BucketId& bucket, - const distributor::BucketInfo& entry) + const BucketInfo& entry) { for (std::size_t i = 0; i < entry.getNodeCount(); ++i) { - const distributor::BucketCopy& copy = entry.getNodeRef(i); + const BucketCopy& copy = entry.getNodeRef(i); if (!copy.valid() && copy.trusted()) { LOG(warning, "Bucket DB entry %s for %s has invalid copy %s " "marked as trusted", entry.toString().c_str(), @@ -305,7 +305,7 @@ checkInvalidImpliesNotTrusted( } void -logBucketDbInsert(uint64_t key, const distributor::BucketInfo& entry) +logBucketDbInsert(uint64_t key, const BucketInfo& entry) { document::BucketId bucket(document::BucketId::keyToBucketId(key)); LOG_BUCKET_OPERATION_NO_LOCK( @@ -318,7 +318,7 @@ logBucketDbInsert(uint64_t key, const distributor::BucketInfo& entry) } void -logBucketDbErase(uint64_t key, const TypeTag<distributor::BucketInfo>&) +logBucketDbErase(uint64_t key, const TypeTag<BucketInfo>&) { document::BucketId bucket(document::BucketId::keyToBucketId(key)); LOG_BUCKET_OPERATION_NO_LOCK(bucket, "bucketdb erase"); diff --git a/storage/src/vespa/storage/common/distributorcomponent.h b/storage/src/vespa/storage/common/distributorcomponent.h index 586bbf61890..a1e28c410d7 100644 --- a/storage/src/vespa/storage/common/distributorcomponent.h +++ b/storage/src/vespa/storage/common/distributorcomponent.h @@ -27,8 +27,8 @@ #include <vespa/storageapi/defs.h> #include <vespa/storage/common/storagecomponent.h> -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> -#include <vespa/storage/distributor/distributorconfiguration.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> +#include <vespa/storage/config/distributorconfiguration.h> #include <vespa/storage/config/config-stor-distributormanager.h> #include <vespa/storage/config/config-stor-visitordispatcher.h> @@ -55,7 +55,7 @@ struct DistributorManagedComponent virtual void setIdealNodeCalculator(lib::IdealNodeCalculator&) = 0; virtual void setTimeCalculator(UniqueTimeCalculator&) = 0; - virtual void setBucketDatabase(distributor::BucketDatabase&) = 0; + virtual void setBucketDatabase(BucketDatabase&) = 0; virtual void setDistributorConfig(const DistributorConfig&)= 0; virtual void setVisitorConfig(const VisitorConfig&) = 0; }; @@ -70,14 +70,14 @@ class DistributorComponent : public StorageComponent, private DistributorManagedComponent { lib::IdealNodeCalculator* _idealNodeCalculator; - distributor::BucketDatabase* _bucketDatabase; + BucketDatabase* _bucketDatabase; mutable UniqueTimeCalculator* _timeCalculator; DistributorConfig _distributorConfig; VisitorConfig _visitorConfig; - distributor::DistributorConfiguration _totalConfig; + DistributorConfiguration _totalConfig; // DistributorManagedComponent implementation - virtual void setBucketDatabase(distributor::BucketDatabase& db) + virtual void setBucketDatabase(BucketDatabase& db) { _bucketDatabase = &db; } virtual void setIdealNodeCalculator(lib::IdealNodeCalculator& c) { _idealNodeCalculator = &c; } @@ -109,11 +109,11 @@ public: const VisitorConfig& getVisitorConfig() const { return _visitorConfig; } - const distributor::DistributorConfiguration& + const DistributorConfiguration& getTotalDistributorConfig() const { return _totalConfig; } - distributor::BucketDatabase& getBucketDatabase() { + BucketDatabase& getBucketDatabase() { assert(_bucketDatabase); return *_bucketDatabase; } lib::IdealNodeCalculator& getIdealNodeCalculator() const { diff --git a/storage/src/vespa/storage/config/CMakeLists.txt b/storage/src/vespa/storage/config/CMakeLists.txt index 863b29e6fd6..8a62f68b27c 100644 --- a/storage/src/vespa/storage/config/CMakeLists.txt +++ b/storage/src/vespa/storage/config/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(storage_storageconfig OBJECT SOURCES + distributorconfiguration.cpp DEPENDS ) vespa_generate_config(storage_storageconfig stor-communicationmanager.def) diff --git a/storage/src/vespa/storage/distributor/distributorconfiguration.cpp b/storage/src/vespa/storage/config/distributorconfiguration.cpp index 0ac1851fdc4..51909ab335a 100644 --- a/storage/src/vespa/storage/distributor/distributorconfiguration.cpp +++ b/storage/src/vespa/storage/config/distributorconfiguration.cpp @@ -1,5 +1,5 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/storage/distributor/distributorconfiguration.h> +#include <vespa/storage/config/distributorconfiguration.h> #include <vespa/log/log.h> #include <vespa/document/select/parser.h> #include <vespa/document/select/node.h> @@ -10,8 +10,6 @@ LOG_SETUP(".distributorconfiguration"); namespace storage { -namespace distributor { - DistributorConfiguration::DistributorConfiguration(StorageComponent& component) : _component(component), _byteCountSplitLimit(0xffffffff), @@ -171,6 +169,5 @@ DistributorConfiguration::configure(const vespa::config::content::core::StorVisi _maxVisitorsPerNodePerClientVisitor = config.maxvisitorspernodeperclientvisitor; } -} // distributor } // storage diff --git a/storage/src/vespa/storage/distributor/distributorconfiguration.h b/storage/src/vespa/storage/config/distributorconfiguration.h index efc57fe534b..0ca2eb49c8a 100644 --- a/storage/src/vespa/storage/distributor/distributorconfiguration.h +++ b/storage/src/vespa/storage/config/distributorconfiguration.h @@ -7,8 +7,10 @@ #include <vespa/storage/common/storagecomponent.h> namespace storage { -namespace distributor { +namespace distributor { + class Distributor_Test; +} class DistributorConfiguration { public: @@ -263,7 +265,7 @@ private: DistrConfig::MinimumReplicaCountingMode _minimumReplicaCountingMode; - friend class Distributor_Test; + friend class distributor::Distributor_Test; bool containsTimeStatement(const std::string& documentSelection) const; void configureMaintenancePriorities( @@ -271,6 +273,5 @@ private: }; } -} diff --git a/storage/src/vespa/storage/distributor/CMakeLists.txt b/storage/src/vespa/storage/distributor/CMakeLists.txt index eb7850c3439..5333c2281f8 100644 --- a/storage/src/vespa/storage/distributor/CMakeLists.txt +++ b/storage/src/vespa/storage/distributor/CMakeLists.txt @@ -17,7 +17,6 @@ vespa_add_library(storage_distributor nodeinfo.cpp sentmessagemap.cpp operationtargetresolverimpl.cpp - distributorconfiguration.cpp throttlingoperationstarter.cpp blockingoperationstarter.cpp distributormessagesender.cpp @@ -27,9 +26,14 @@ vespa_add_library(storage_distributor bucketgctimecalculator.cpp distributor_host_info_reporter.cpp latency_statistics_provider.cpp - $<TARGET_OBJECTS:storage_bucketdatabase> + $<TARGET_OBJECTS:storage_distributoroperation> + $<TARGET_OBJECTS:storage_distributoroperationexternal> + $<TARGET_OBJECTS:storage_distributoroperationidealstate> + $<TARGET_OBJECTS:storage_distributormaintenance> + $<TARGET_OBJECTS:storage_distributor_bucketdb> INSTALL lib64 DEPENDS + storage AFTER storage_storageconfig ) diff --git a/storage/src/vespa/storage/distributor/activecopy.h b/storage/src/vespa/storage/distributor/activecopy.h index 73e135af352..00a7bf3d4f6 100644 --- a/storage/src/vespa/storage/distributor/activecopy.h +++ b/storage/src/vespa/storage/distributor/activecopy.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> namespace storage { namespace lib { diff --git a/storage/src/vespa/storage/distributor/bucketdb/CMakeLists.txt b/storage/src/vespa/storage/distributor/bucketdb/CMakeLists.txt index b9f5f926a65..160f1d2ad17 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/CMakeLists.txt +++ b/storage/src/vespa/storage/distributor/bucketdb/CMakeLists.txt @@ -1,11 +1,6 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_bucketdatabase OBJECT +vespa_add_library(storage_distributor_bucketdb OBJECT SOURCES - bucketinfo.cpp - bucketcopy.cpp - bucketdatabase.cpp - judybucketdatabase.cpp - mapbucketdatabase.cpp bucketdbmetricupdater.cpp DEPENDS AFTER diff --git a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h index 1fb7b5672f5..9f59426f983 100644 --- a/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdb/bucketdbmetricupdater.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storage/config/config-stor-distributormanager.h> #include <unordered_map> diff --git a/storage/src/vespa/storage/distributor/bucketdb/judybucketdatabase.cpp b/storage/src/vespa/storage/distributor/bucketdb/judybucketdatabase.cpp deleted file mode 100644 index 62dc47f0881..00000000000 --- a/storage/src/vespa/storage/distributor/bucketdb/judybucketdatabase.cpp +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/fastos.h> -#include <vespa/storage/distributor/bucketdb/judybucketdatabase.h> - -namespace storage { - -using bucketdb::DistrBucketDatabase; - -namespace distributor { - -BucketDatabase::Entry -JudyBucketDatabase::get(const document::BucketId& bucket) const -{ - DistrBucketDatabase::WrappedEntry wrp = _db.get(bucket, "", false); - if (!wrp.exist()) { - return BucketDatabase::Entry(); - } else { - return BucketDatabase::Entry(bucket, *wrp); - } -} - -void -JudyBucketDatabase::remove(const document::BucketId& bucket) -{ - DistrBucketDatabase::WrappedEntry wrp = _db.get(bucket, "", false); - if (wrp.exist()) { - wrp.remove(); - } -} - -void -JudyBucketDatabase::getParents(const document::BucketId& childBucket, - std::vector<Entry>& entries) const -{ - bucketdb::DistrBucketDatabase::EntryMap e = _db.getContained(childBucket, ""); - - for (bucketdb::DistrBucketDatabase::EntryMap::iterator iter = e.begin(); - iter != e.end(); - ++iter) { - entries.push_back(BucketDatabase::Entry(iter->first, - *iter->second)); - } -} - -void -JudyBucketDatabase::getAll(const document::BucketId& bucket, - std::vector<Entry>& entries) const -{ - bucketdb::DistrBucketDatabase::EntryMap e = _db.getAll(bucket, ""); - - for (bucketdb::DistrBucketDatabase::EntryMap::iterator iter = e.begin(); - iter != e.end(); - ++iter) { - entries.push_back(BucketDatabase::Entry(iter->first, - *iter->second)); - } -} - -void -JudyBucketDatabase::update(const Entry& newEntry) -{ - DistrBucketDatabase::WrappedEntry wrp = _db.get(newEntry.getBucketId(), "", true); - (*wrp) = newEntry.getBucketInfo(); - wrp.write(); -} - -namespace { - -class Iterator { -public: - document::BucketId lastBucketId; - BucketDatabase::Entry lastEntry; - - - Iterator(const document::BucketId& b) - : lastBucketId(b) {}; - - DistrBucketDatabase::Decision operator()(document::BucketId::Type key, - DistrBucketDatabase::Entry& info) - { - document::BucketId bucketId(document::BucketId::keyToBucketId(key)); - - if (lastBucketId == bucketId) { - return DistrBucketDatabase::CONTINUE; - } - - lastEntry = BucketDatabase::Entry(bucketId, info); - return DistrBucketDatabase::ABORT; - } - -}; - -} - -void -JudyBucketDatabase::forEach(EntryProcessor& processor, - const document::BucketId& last) const -{ - document::BucketId curr = last; - - JudyBucketDatabase& mutableSelf(const_cast<JudyBucketDatabase&>(*this)); - Entry currEntry; - while ((currEntry = mutableSelf.getNextEntry(curr)).valid()) { - - bool continueProcessing = processor.process(currEntry); - if (!continueProcessing) { - break; - } - curr = currEntry.getBucketId(); - } -} - -BucketDatabase::Entry -JudyBucketDatabase::getNextEntry(const document::BucketId& curr) -{ - return upperBound(curr); -} - -void -JudyBucketDatabase::forEach(MutableEntryProcessor& processor, - const document::BucketId& last) -{ - document::BucketId curr = last; - - Entry currEntry; - while ((currEntry = getNextEntry(curr)).valid()) { - - Entry lastEntry = currEntry; - bool continueProcessing = processor.process(currEntry); - if (!(currEntry.getBucketInfo() == lastEntry.getBucketInfo())) { - update(currEntry); - } - - if (!continueProcessing) { - break; - } - curr = currEntry.getBucketId(); - } -} - -uint64_t -JudyBucketDatabase::size() const -{ - return _db.size(); -} - -void -JudyBucketDatabase::clear() -{ - _db.clear(); -} - -// FIXME: mutates database! No read-only functionality for this in LocakableMap! -document::BucketId -JudyBucketDatabase::getAppropriateBucket( - uint16_t minBits, - const document::BucketId& bid) -{ - DistrBucketDatabase::WrappedEntry wrp = - _db.createAppropriateBucket(minBits, "", bid); - return wrp.getBucketId(); -} - -uint32_t -JudyBucketDatabase::childCount(const document::BucketId&) const -{ - // Not implemented! Judy map for distributor is deprecated. - abort(); -} - -BucketDatabase::Entry -JudyBucketDatabase::upperBound(const document::BucketId& value) const -{ - Iterator iter(value); - _db.all(iter, "", value.toKey()); - return iter.lastEntry; -} - -void -JudyBucketDatabase::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) out; (void) verbose; (void) indent; -} - -} -} diff --git a/storage/src/vespa/storage/distributor/bucketdb/judybucketdatabase.h b/storage/src/vespa/storage/distributor/bucketdb/judybucketdatabase.h deleted file mode 100644 index 13cddd72cdf..00000000000 --- a/storage/src/vespa/storage/distributor/bucketdb/judybucketdatabase.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> - -namespace storage { -namespace distributor { - -class JudyBucketDatabase : public BucketDatabase -{ -public: - virtual Entry get(const document::BucketId& bucket) const; - virtual void remove(const document::BucketId& bucket); - virtual void getParents(const document::BucketId& childBucket, - std::vector<Entry>& entries) const; - virtual void getAll(const document::BucketId& bucket, - std::vector<Entry>& entries) const; - virtual void update(const Entry& newEntry); - virtual void forEach(EntryProcessor&, - const document::BucketId& after) const; - virtual void forEach(MutableEntryProcessor&, - const document::BucketId& after); - uint64_t size() const; - void clear(); - - // FIXME: remove! mutates internal database! - document::BucketId getAppropriateBucket( - uint16_t minBits, - const document::BucketId& bid); - - uint32_t childCount(const document::BucketId&) const override; - - Entry upperBound(const document::BucketId& bucket) const override; - - virtual void print(std::ostream& out, bool verbose, - const std::string& indent) const; - -private: - mutable bucketdb::DistrBucketDatabase _db; - - Entry getNextEntry(const document::BucketId& id); -}; - -} -} - diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp index 6ecb8dd9276..b7235c4f13d 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.cpp +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.cpp @@ -6,7 +6,6 @@ #include <vespa/storageapi/message/state.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/log/log.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageapi/message/removelocation.h> #include <vespa/storageapi/message/multioperation.h> diff --git a/storage/src/vespa/storage/distributor/bucketdbupdater.h b/storage/src/vespa/storage/distributor/bucketdbupdater.h index 01fae03f44d..6c50809184e 100644 --- a/storage/src/vespa/storage/distributor/bucketdbupdater.h +++ b/storage/src/vespa/storage/distributor/bucketdbupdater.h @@ -6,7 +6,6 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/storageapi/messageapi/returncode.h> #include <vespa/storageapi/message/bucket.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/common/storagelink.h> #include <vespa/storageframework/storageframework.h> diff --git a/storage/src/vespa/storage/distributor/distributor.cpp b/storage/src/vespa/storage/distributor/distributor.cpp index 2c806901de5..3cdf5926c22 100644 --- a/storage/src/vespa/storage/distributor/distributor.cpp +++ b/storage/src/vespa/storage/distributor/distributor.cpp @@ -2,7 +2,7 @@ #include <vespa/fastos/fastos.h> #include <vespa/log/log.h> #include <vespa/storage/distributor/distributor.h> -#include <vespa/storage/distributor/bucketdb/mapbucketdatabase.h> +#include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <vespa/storage/distributor/maintenance/simplemaintenancescanner.h> #include <vespa/storage/distributor/maintenance/simplebucketprioritydatabase.h> #include <vespa/storage/distributor/blockingoperationstarter.h> diff --git a/storage/src/vespa/storage/distributor/distributorcomponent.cpp b/storage/src/vespa/storage/distributor/distributorcomponent.cpp index a8e5a380b8e..a2cca6a830b 100644 --- a/storage/src/vespa/storage/distributor/distributorcomponent.cpp +++ b/storage/src/vespa/storage/distributor/distributorcomponent.cpp @@ -4,7 +4,6 @@ #include <vespa/log/log.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/storageapi/messageapi/storagereply.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/common/bucketoperationlogger.h> LOG_SETUP(".distributorstoragelink"); diff --git a/storage/src/vespa/storage/distributor/distributorcomponent.h b/storage/src/vespa/storage/distributor/distributorcomponent.h index fbf773e7fff..9efa4c5186b 100644 --- a/storage/src/vespa/storage/distributor/distributorcomponent.h +++ b/storage/src/vespa/storage/distributor/distributorcomponent.h @@ -6,7 +6,6 @@ #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storageapi/messageapi/storagecommand.h> #include <vespa/storageapi/buckets/bucketinfo.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/common/distributorcomponent.h> #include <vespa/storage/distributor/statechecker.h> #include <vespa/storage/storageutil/utils.h> diff --git a/storage/src/vespa/storage/distributor/distributorinterface.h b/storage/src/vespa/storage/distributor/distributorinterface.h index 5562670c35e..ecbc0163b01 100644 --- a/storage/src/vespa/storage/distributor/distributorinterface.h +++ b/storage/src/vespa/storage/distributor/distributorinterface.h @@ -6,10 +6,10 @@ #include <vespa/storage/distributor/pendingmessagetracker.h> #include <vespa/storageapi/message/state.h> #include <vespa/storage/distributor/maintenancebucket.h> -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storage/distributor/bucketgctimecalculator.h> #include <vespa/storage/distributor/distributormetricsset.h> -#include <vespa/storage/distributor/distributorconfiguration.h> +#include <vespa/storage/config/distributorconfiguration.h> #include <vespa/storage/distributor/distributormessagesender.h> #include <vespa/storage/distributor/bucketownership.h> diff --git a/storage/src/vespa/storage/distributor/externaloperationhandler.h b/storage/src/vespa/storage/distributor/externaloperationhandler.h index f8c88c31604..0541faa7368 100644 --- a/storage/src/vespa/storage/distributor/externaloperationhandler.h +++ b/storage/src/vespa/storage/distributor/externaloperationhandler.h @@ -4,7 +4,6 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/document/bucket/bucketidfactory.h> #include <vespa/vdslib/state/clusterstate.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/distributorcomponent.h> #include <vespa/storage/distributor/visitormetricsset.h> #include <vespa/storageapi/messageapi/messagehandler.h> diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.h b/storage/src/vespa/storage/distributor/idealstatemanager.h index 7dba4102466..0fc29cf834a 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.h +++ b/storage/src/vespa/storage/distributor/idealstatemanager.h @@ -4,7 +4,6 @@ #include <deque> #include <map> #include <set> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/distributorcomponent.h> #include <vespa/storage/distributor/statechecker.h> #include <vespa/storage/distributor/maintenance/maintenanceprioritygenerator.h> diff --git a/storage/src/vespa/storage/distributor/maintenance/bucketprioritydatabase.h b/storage/src/vespa/storage/distributor/maintenance/bucketprioritydatabase.h index 3a4a278543f..30ccf0040ba 100644 --- a/storage/src/vespa/storage/distributor/maintenance/bucketprioritydatabase.h +++ b/storage/src/vespa/storage/distributor/maintenance/bucketprioritydatabase.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storage/distributor/maintenance/prioritizedbucket.h> #include <boost/iterator/iterator_facade.hpp> #include <vespa/vespalib/util/linkedptr.h> diff --git a/storage/src/vespa/storage/distributor/maintenance/maintenancescanner.h b/storage/src/vespa/storage/distributor/maintenance/maintenancescanner.h index 9dce66c1f32..50d947008de 100644 --- a/storage/src/vespa/storage/distributor/maintenance/maintenancescanner.h +++ b/storage/src/vespa/storage/distributor/maintenance/maintenancescanner.h @@ -1,7 +1,7 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> namespace storage { namespace distributor { diff --git a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h index eaec4d5186b..b62ecffa0fd 100644 --- a/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h +++ b/storage/src/vespa/storage/distributor/maintenance/simplemaintenancescanner.h @@ -7,7 +7,7 @@ #include <vespa/storage/distributor/maintenance/bucketprioritydatabase.h> #include <vespa/storage/distributor/maintenance/maintenanceprioritygenerator.h> #include <vespa/storage/distributor/maintenance/node_maintenance_stats_tracker.h> -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> namespace storage { namespace distributor { diff --git a/storage/src/vespa/storage/distributor/messagetracker.h b/storage/src/vespa/storage/distributor/messagetracker.h index eb41a7ccd27..991169b7ae6 100644 --- a/storage/src/vespa/storage/distributor/messagetracker.h +++ b/storage/src/vespa/storage/distributor/messagetracker.h @@ -4,7 +4,6 @@ #include <vespa/storageapi/messageapi/bucketcommand.h> #include <vespa/storageapi/messageapi/bucketreply.h> #include <vespa/storage/distributor/distributormetricsset.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/common/messagesender.h> namespace storage { diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp index 883fd3809d6..20988bcee27 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.cpp @@ -4,7 +4,6 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/log/log.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/distributorcomponent.h> #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/vdslib/state/nodestate.h> diff --git a/storage/src/vespa/storage/distributor/operations/external/getoperation.h b/storage/src/vespa/storage/distributor/operations/external/getoperation.h index 6af02abe144..3a0c1dbebc3 100644 --- a/storage/src/vespa/storage/distributor/operations/external/getoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/getoperation.h @@ -3,7 +3,7 @@ #include <vespa/storageapi/defs.h> #include <vespa/storage/distributor/operations/operation.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> +#include <vespa/storage/bucketdb/bucketcopy.h> #include <vespa/storageapi/messageapi/storagemessage.h> namespace document { diff --git a/storage/src/vespa/storage/distributor/operations/external/multioperationoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/multioperationoperation.cpp index ab4bdfbd3dd..9d056eb0fd0 100644 --- a/storage/src/vespa/storage/distributor/operations/external/multioperationoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/multioperationoperation.cpp @@ -2,7 +2,6 @@ #include <vespa/fastos/fastos.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/log/log.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storage/distributor/operations/external/multioperationoperation.h> #include <vespa/storage/distributor/operations/external/putoperation.h> diff --git a/storage/src/vespa/storage/distributor/operations/external/multioperationoperation.h b/storage/src/vespa/storage/distributor/operations/external/multioperationoperation.h index 95f31537dfb..20d64e95fae 100644 --- a/storage/src/vespa/storage/distributor/operations/external/multioperationoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/multioperationoperation.h @@ -4,7 +4,7 @@ #include <vespa/storage/distributor/persistencemessagetracker.h> #include <vespa/storageapi/messageapi/returncode.h> #include <vespa/vdslib/container/writabledocumentlist.h> -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> namespace document { class Document; diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp index 51198043775..f4a2f9f0307 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp @@ -4,7 +4,6 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/log/log.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/activecopy.h> #include <vespa/storage/distributor/distributorcomponent.h> #include <vespa/storage/distributor/distributormetricsset.h> diff --git a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp index 5adaa2b8cc1..9726200b588 100644 --- a/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/removelocationoperation.cpp @@ -5,7 +5,6 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/select/parser.h> #include <vespa/log/log.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storage/distributor/operations/external/removelocationoperation.h> #include <vespa/storageapi/message/removelocation.h> diff --git a/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp index b6d575071a2..596debf54a6 100644 --- a/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/removeoperation.cpp @@ -4,7 +4,6 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/log/log.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/distributormetricsset.h> LOG_SETUP(".distributor.operation.external.remove"); diff --git a/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h b/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h index a1b4eb110b7..8e092a96f88 100644 --- a/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/statbucketlistoperation.h @@ -2,7 +2,7 @@ #pragma once #include <vespa/storage/distributor/operations/operation.h> -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/vespalib/util/sync.h> namespace storage { diff --git a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp index b3a7ec4805e..c54598b7a76 100644 --- a/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/statbucketoperation.cpp @@ -4,7 +4,6 @@ #include <vespa/log/log.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/distributorcomponent.h> #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storageapi/message/stat.h> diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 72a6dede161..656ad446da7 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -5,7 +5,6 @@ #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/log/log.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storage/distributor/operations/external/getoperation.h> #include <vespa/storage/distributor/operations/external/putoperation.h> diff --git a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp index 57a5c968d54..d2b29a092bc 100644 --- a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp @@ -5,7 +5,6 @@ #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/log/log.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/distributormetricsset.h> LOG_SETUP(".distributor.callback.doc.update"); diff --git a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h index 6de20b81f25..a0aac7918e6 100644 --- a/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h +++ b/storage/src/vespa/storage/distributor/operations/external/visitoroperation.h @@ -3,11 +3,10 @@ #include <vespa/storageapi/defs.h> #include <vespa/storage/distributor/operations/operation.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/storageapi/message/visitor.h> #include <vespa/storage/distributor/visitormetricsset.h> -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storage/visiting/memory_bounded_trace.h> namespace document { diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp index 2deab309ea4..e3430f9fa0e 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.cpp @@ -3,7 +3,6 @@ #include <vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h> #include <vespa/storageapi/messageapi/storagereply.h> #include <vespa/storageapi/message/bucket.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/pendingmessagetracker.h> diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h index 29e9aa7422c..3491eacf066 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/idealstateoperation.h @@ -6,7 +6,6 @@ #include <vespa/storageapi/messageapi/storagereply.h> #include <vespa/storageapi/messageapi/maintenancecommand.h> #include <vespa/document/bucket/bucketid.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storageframework/generic/memory/memorymanagerinterface.h> namespace storage diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp index bc7322b579a..5c1f906aae7 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/joinoperation.cpp @@ -4,7 +4,6 @@ #include <vespa/storage/distributor/operations/idealstate/joinoperation.h> #include <vespa/storageapi/messageapi/storagereply.h> #include <vespa/storageapi/message/bucketsplitting.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/distributor/pendingmessagetracker.h> diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergemetadata.h b/storage/src/vespa/storage/distributor/operations/idealstate/mergemetadata.h index 3edc9a3f268..77f3ff6b678 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergemetadata.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergemetadata.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/storage/distributor/bucketdb/bucketcopy.h> +#include <vespa/storage/bucketdb/bucketcopy.h> #include <vespa/vespalib/stllike/asciistream.h> namespace storage { diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp index e448192abfc..f222738cd9a 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.cpp @@ -3,7 +3,6 @@ #include <vespa/storage/distributor/operations/idealstate/mergeoperation.h> #include <vespa/storageapi/messageapi/storagereply.h> #include <vespa/storageapi/message/bucket.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/operations/idealstate/removebucketoperation.h> diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h index 0dfe7756b79..b99a302a9d3 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h +++ b/storage/src/vespa/storage/distributor/operations/idealstate/mergeoperation.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storage/distributor/operations/idealstate/idealstateoperation.h> #include <vespa/storage/distributor/operations/idealstate/mergelimiter.h> #include <vespa/storage/distributor/operations/idealstate/mergemetadata.h> diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp index 0db8cda367a..3d0a27f5215 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp @@ -3,7 +3,6 @@ #include <vespa/storage/distributor/operations/idealstate/removebucketoperation.h> #include <vespa/storageapi/messageapi/storagereply.h> #include <vespa/storageapi/message/bucket.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/distributor/distributor.h> #include <vespa/storage/distributor/pendingmessagetracker.h> diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp index 423039f2b0c..0a37a1d1e37 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/splitoperation.cpp @@ -3,7 +3,6 @@ #include <vespa/storage/distributor/operations/idealstate/splitoperation.h> #include <vespa/storageapi/messageapi/storagereply.h> #include <vespa/storageapi/message/bucketsplitting.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/distributor/idealstatemanager.h> #include <vespa/storage/common/bucketoperationlogger.h> diff --git a/storage/src/vespa/storage/distributor/operationtargetresolver.cpp b/storage/src/vespa/storage/distributor/operationtargetresolver.cpp index 05255eefcb6..12cf8e0ca85 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolver.cpp +++ b/storage/src/vespa/storage/distributor/operationtargetresolver.cpp @@ -5,7 +5,6 @@ #include <queue> #include <vespa/storage/distributor/operationtargetresolver.h> #include <vespa/storage/distributor/bucketdb/bucketdatabase.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/vespalib/stllike/hash_set.h> namespace storage { diff --git a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h index 36cb02dc213..25ce4be09dc 100644 --- a/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h +++ b/storage/src/vespa/storage/distributor/operationtargetresolverimpl.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storage/distributor/operationtargetresolver.h> #include <vespa/vdslib/distribution/idealnodecalculator.h> diff --git a/storage/src/vespa/storage/distributor/pendingclusterstate.h b/storage/src/vespa/storage/distributor/pendingclusterstate.h index c31747cf600..5ea95c2b3e4 100644 --- a/storage/src/vespa/storage/distributor/pendingclusterstate.h +++ b/storage/src/vespa/storage/distributor/pendingclusterstate.h @@ -4,10 +4,9 @@ #include <vespa/storage/common/storagelink.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/util/xmlserializable.h> -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storage/distributor/clusterinformation.h> #include <unordered_set> diff --git a/storage/src/vespa/storage/distributor/persistencemessagetracker.h b/storage/src/vespa/storage/distributor/persistencemessagetracker.h index 3f667c146eb..b5c3e57e05e 100644 --- a/storage/src/vespa/storage/distributor/persistencemessagetracker.h +++ b/storage/src/vespa/storage/distributor/persistencemessagetracker.h @@ -2,7 +2,6 @@ #pragma once #include <vespa/storage/distributor/distributormetricsset.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/distributor/distributorcomponent.h> #include <vespa/storage/distributor/messagetracker.h> #include <vespa/storageapi/messageapi/bucketinfocommand.h> diff --git a/storage/src/vespa/storage/distributor/statechecker.h b/storage/src/vespa/storage/distributor/statechecker.h index 3b0d16644df..48f43cbd00f 100644 --- a/storage/src/vespa/storage/distributor/statechecker.h +++ b/storage/src/vespa/storage/distributor/statechecker.h @@ -1,14 +1,13 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/common/storagecomponent.h> #include <vespa/storage/distributor/operations/idealstate/idealstateoperation.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/storage/distributor/bucketgctimecalculator.h> #include <vespa/storage/distributor/maintenancebucket.h> -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/vespalib/util/linkedptr.h> #include <unordered_set> @@ -17,10 +16,11 @@ namespace storage { +class DistributorConfiguration; + namespace distributor { class DistributorComponent; -class DistributorConfiguration; class NodeMaintenanceStatsTracker; /** diff --git a/storage/src/vespa/storage/distributor/statecheckers.cpp b/storage/src/vespa/storage/distributor/statecheckers.cpp index e93806e2181..dd247eb497e 100644 --- a/storage/src/vespa/storage/distributor/statecheckers.cpp +++ b/storage/src/vespa/storage/distributor/statecheckers.cpp @@ -4,13 +4,13 @@ #include <vespa/log/log.h> #include <vespa/storage/distributor/activecopy.h> -#include <vespa/storage/distributor/bucketdb/bucketdatabase.h> #include <vespa/storage/distributor/operations/idealstate/splitoperation.h> #include <vespa/storage/distributor/operations/idealstate/joinoperation.h> #include <vespa/storage/distributor/operations/idealstate/removebucketoperation.h> #include <vespa/storage/distributor/operations/idealstate/setbucketstateoperation.h> #include <vespa/storage/distributor/operations/idealstate/mergeoperation.h> #include <vespa/storage/distributor/operations/idealstate/garbagecollectionoperation.h> +#include <vespa/storage/bucketdb/bucketdatabase.h> #include <vespa/storage/common/bucketoperationlogger.h> #include <vespa/vespalib/stllike/asciistream.h> diff --git a/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h b/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h index d1a37c5ba5e..d70d69f6022 100644 --- a/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h +++ b/storage/src/vespa/storage/frameworkimpl/component/distributorcomponentregisterimpl.h @@ -7,7 +7,7 @@ */ #pragma once -#include <vespa/storage/distributor/bucketdb/mapbucketdatabase.h> +#include <vespa/storage/bucketdb/mapbucketdatabase.h> #include <vespa/storage/common/distributorcomponent.h> #include <vespa/storage/common/nodestateupdater.h> #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> @@ -26,7 +26,7 @@ class DistributorComponentRegisterImpl std::vector<DistributorManagedComponent*> _components; UniqueTimeCalculator* _timeCalculator; - distributor::MapBucketDatabase _bucketDatabase; + MapBucketDatabase _bucketDatabase; DistributorConfig _distributorConfig; VisitorConfig _visitorConfig; lib::ClusterState _clusterState; @@ -37,7 +37,7 @@ public: DistributorComponentRegisterImpl(); - distributor::BucketDatabase& getBucketDatabase() { return _bucketDatabase; } + BucketDatabase& getBucketDatabase() { return _bucketDatabase; } virtual void registerDistributorComponent(DistributorManagedComponent&); diff --git a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp index 50f9a37e55e..99b481877ca 100644 --- a/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp +++ b/storage/src/vespa/storage/frameworkimpl/thread/deadlockdetector.cpp @@ -4,7 +4,6 @@ #include <vespa/storage/frameworkimpl/thread/deadlockdetector.h> #include <vespa/log/log.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storageframework/storageframework.h> #include <vespa/vdslib/state/nodetype.h> diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 2253fb1dcee..17638f55b3a 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_storageserver OBJECT +vespa_add_library(storage_storageserver SOURCES priorityconverter.cpp bucketintegritychecker.cpp @@ -21,8 +21,8 @@ vespa_add_library(storage_storageserver OBJECT servicelayernode.cpp statereporter.cpp changedbucketownershiphandler.cpp + INSTALL lib64 DEPENDS - AFTER - storage_storageconfig - storage_visitor + storage + storage_distributor ) diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index 8afe3397179..7429283808c 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -7,9 +7,7 @@ * * This class sets up the entire storage server. * - * @author H?kon Humberset - * @date 2005-05-13 - * @version $Id: storageserver.h 131081 2011-12-16 18:44:06Z lulf $ + * @author Håkon Humberset */ #pragma once @@ -25,7 +23,6 @@ #include <vespa/document/config/config-documenttypes.h> #include <vespa/documentapi/loadtypes/loadtypeset.h> #include <vespa/metrics/metrics.h> -#include <vespa/storage/bucketdb/distrbucketdb.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/common/doneinitializehandler.h> #include <vespa/storage/common/storagelink.h> diff --git a/storageapi/src/tests/buckets/CMakeLists.txt b/storageapi/src/tests/buckets/CMakeLists.txt index aa25f630f53..5e50945bb19 100644 --- a/storageapi/src/tests/buckets/CMakeLists.txt +++ b/storageapi/src/tests/buckets/CMakeLists.txt @@ -3,5 +3,6 @@ vespa_add_library(storageapi_testbuckets SOURCES bucketinfotest.cpp DEPENDS + storageapi cppunit ) diff --git a/storageapi/src/tests/mbusprot/CMakeLists.txt b/storageapi/src/tests/mbusprot/CMakeLists.txt index d7d376ae105..75a0bddc0bd 100644 --- a/storageapi/src/tests/mbusprot/CMakeLists.txt +++ b/storageapi/src/tests/mbusprot/CMakeLists.txt @@ -3,5 +3,6 @@ vespa_add_library(storageapi_testmbusprot SOURCES storageprotocoltest.cpp DEPENDS + storageapi vdstestlib ) diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp b/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp index 0165348968e..f91893ea355 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp +++ b/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp @@ -4,7 +4,6 @@ #include <vespa/storageapi/mbusprot/storagecommand.h> -using vespalib::DefaultAlloc; using vespalib::alloc::Alloc; using vespalib::IllegalStateException; @@ -14,7 +13,7 @@ namespace mbusprot { StorageReply::StorageReply(const mbus::BlobRef& data, const ProtocolSerialization& serializer) : _serializer(&serializer), - _buffer(DefaultAlloc::create(data.size())), + _buffer(Alloc::alloc(data.size())), _mbusType(0), _reply() { diff --git a/storageframework/src/tests/CMakeLists.txt b/storageframework/src/tests/CMakeLists.txt index 47a0a7bcc6e..d96113597fd 100644 --- a/storageframework/src/tests/CMakeLists.txt +++ b/storageframework/src/tests/CMakeLists.txt @@ -7,6 +7,5 @@ vespa_add_executable(storageframework_testrunner_app TEST storageframework_teststatus storageframework_testmemory storageframework_testthread - storageframework ) vespa_add_test(NAME storageframework_testrunner_app COMMAND storageframework_testrunner_app) diff --git a/storageframework/src/tests/clock/CMakeLists.txt b/storageframework/src/tests/clock/CMakeLists.txt index 1134cf8e88a..58b3c81d6eb 100644 --- a/storageframework/src/tests/clock/CMakeLists.txt +++ b/storageframework/src/tests/clock/CMakeLists.txt @@ -3,4 +3,5 @@ vespa_add_library(storageframework_testclock SOURCES timetest.cpp DEPENDS + storageframework ) diff --git a/storageframework/src/tests/memory/CMakeLists.txt b/storageframework/src/tests/memory/CMakeLists.txt index 97a7314f4cb..0905ce19aca 100644 --- a/storageframework/src/tests/memory/CMakeLists.txt +++ b/storageframework/src/tests/memory/CMakeLists.txt @@ -4,4 +4,5 @@ vespa_add_library(storageframework_testmemory memorymanagertest.cpp memorystatetest.cpp DEPENDS + storageframework ) diff --git a/storageframework/src/tests/status/CMakeLists.txt b/storageframework/src/tests/status/CMakeLists.txt index 2c829d48810..7b1cefec6ad 100644 --- a/storageframework/src/tests/status/CMakeLists.txt +++ b/storageframework/src/tests/status/CMakeLists.txt @@ -3,4 +3,5 @@ vespa_add_library(storageframework_teststatus SOURCES htmlstatustest.cpp DEPENDS + storageframework ) diff --git a/storageframework/src/tests/thread/CMakeLists.txt b/storageframework/src/tests/thread/CMakeLists.txt index 961b49d065a..904a0441c99 100644 --- a/storageframework/src/tests/thread/CMakeLists.txt +++ b/storageframework/src/tests/thread/CMakeLists.txt @@ -4,4 +4,5 @@ vespa_add_library(storageframework_testthread tickingthreadtest.cpp taskthreadtest.cpp DEPENDS + storageframework ) diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt b/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt index 51e992b134d..e0eafa63872 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt +++ b/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt @@ -7,4 +7,5 @@ vespa_add_library(storageframework_defaultimplementation $<TARGET_OBJECTS:storageframework_threadimpl> INSTALL lib64 DEPENDS + storageframework_generic ) diff --git a/storageserver/src/tests/storageservertest.cpp b/storageserver/src/tests/storageservertest.cpp index 69c55248eb7..379a8c64d78 100644 --- a/storageserver/src/tests/storageservertest.cpp +++ b/storageserver/src/tests/storageservertest.cpp @@ -173,7 +173,7 @@ namespace { virtual StorageNode& getNode() { return _process.getNode(); } virtual StorageNodeContext& getContext() { return _process.getContext(); } - distributor::BucketDatabase& getBucketDatabase() + BucketDatabase& getBucketDatabase() { return _process.getDistributorContext().getComponentRegister().getBucketDatabase(); } }; @@ -673,7 +673,7 @@ StorageServerTest::testSplitJoinSplitThroughDistributor_Stress() } dummyLink.reset(); // Wait until system has split to 7 buckets - distributor::BucketDatabase& db(distServer.getBucketDatabase()); + BucketDatabase& db(distServer.getBucketDatabase()); for (size_t i(0); (i < 6000) && (7ul != db.size()); i++) { FastOS_Thread::Sleep(10); } diff --git a/storageserver/src/vespa/storageserver/app/CMakeLists.txt b/storageserver/src/vespa/storageserver/app/CMakeLists.txt index e2918ae49d6..6c038d65bb2 100644 --- a/storageserver/src/vespa/storageserver/app/CMakeLists.txt +++ b/storageserver/src/vespa/storageserver/app/CMakeLists.txt @@ -8,4 +8,5 @@ vespa_add_library(storageserver_storageapp STATIC rpcservicelayerprocess.cpp memfileservicelayerprocess.cpp DEPENDS + storage_storageserver ) diff --git a/vdslib/src/tests/CMakeLists.txt b/vdslib/src/tests/CMakeLists.txt index b5af2f05501..d2b00ce8e70 100644 --- a/vdslib/src/tests/CMakeLists.txt +++ b/vdslib/src/tests/CMakeLists.txt @@ -7,6 +7,5 @@ vespa_add_executable(vdslib_testrunner_app TEST vdslib_testdistribution vdslib_teststate vdslib_testthread - vdslib ) vespa_add_test(NAME vdslib_testrunner_app COMMAND vdslib_testrunner_app) diff --git a/vdslib/src/tests/bucketdistribution/CMakeLists.txt b/vdslib/src/tests/bucketdistribution/CMakeLists.txt index 511a791d924..ca4cf84f0fb 100644 --- a/vdslib/src/tests/bucketdistribution/CMakeLists.txt +++ b/vdslib/src/tests/bucketdistribution/CMakeLists.txt @@ -3,4 +3,5 @@ vespa_add_library(vdslib_bucketdistributiontest SOURCES bucketdistributiontest.cpp DEPENDS + vdslib ) diff --git a/vdslib/src/tests/container/CMakeLists.txt b/vdslib/src/tests/container/CMakeLists.txt index d8975e1da72..a869d0fd40b 100644 --- a/vdslib/src/tests/container/CMakeLists.txt +++ b/vdslib/src/tests/container/CMakeLists.txt @@ -7,4 +7,5 @@ vespa_add_library(vdslib_containertest documentsummarytest.cpp lruordertest.cpp DEPENDS + vdslib ) diff --git a/vdslib/src/tests/distribution/CMakeLists.txt b/vdslib/src/tests/distribution/CMakeLists.txt index 61aabc1d334..e4197920add 100644 --- a/vdslib/src/tests/distribution/CMakeLists.txt +++ b/vdslib/src/tests/distribution/CMakeLists.txt @@ -5,4 +5,5 @@ vespa_add_library(vdslib_testdistribution grouptest.cpp idealnodecalculatorimpltest.cpp DEPENDS + vdslib ) diff --git a/vdslib/src/tests/state/CMakeLists.txt b/vdslib/src/tests/state/CMakeLists.txt index fead8cd022b..33490374616 100644 --- a/vdslib/src/tests/state/CMakeLists.txt +++ b/vdslib/src/tests/state/CMakeLists.txt @@ -4,4 +4,5 @@ vespa_add_library(vdslib_teststate clusterstatetest.cpp nodestatetest.cpp DEPENDS + vdslib ) diff --git a/vdslib/src/tests/thread/CMakeLists.txt b/vdslib/src/tests/thread/CMakeLists.txt index 210ecae3440..a51d0c588f6 100644 --- a/vdslib/src/tests/thread/CMakeLists.txt +++ b/vdslib/src/tests/thread/CMakeLists.txt @@ -3,4 +3,5 @@ vespa_add_library(vdslib_testthread SOURCES taskschedulertest.cpp DEPENDS + vdslib ) diff --git a/vespajlib/src/main/java/com/yahoo/net/HostName.java b/vespajlib/src/main/java/com/yahoo/net/HostName.java index 4e791ca117a..449688f335b 100644 --- a/vespajlib/src/main/java/com/yahoo/net/HostName.java +++ b/vespajlib/src/main/java/com/yahoo/net/HostName.java @@ -2,40 +2,139 @@ package com.yahoo.net; import java.io.BufferedReader; +import java.io.IOException; import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.net.SocketException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; /** - * Utilities for getting the hostname on a system running with the JVM. This is moved here from the old - * HostSystem#getHostName in config-model. + * Utilities for getting the hostname of the system running the JVM. * * @author lulf + * @author bratseth + * @author hakon */ public class HostName { - private static String myHost = null; + private static final Logger logger = Logger.getLogger(HostName.class.getName()); + + private static String cachedHostName = null; /** - * Static method that returns the name of localhost using shell command "hostname". - * If you need a guaranteed resolvable name see LinuxINetAddress. + * Return a fully qualified hostname that resolves to an IP address on a network interface. + * Normally this is the same as the 'hostname' command, but on dev machines on WiFi, + * that IP isn't configured so we prefer a WiFi network interface IP address which is both reachable and + * has a DNS entry. * - * @return the name of localhost. - * @throws RuntimeException if executing the command 'hostname' fails. + * @return the preferred name of localhost + * @throws RuntimeException if accessing the network or the 'hostname' command fails */ public static synchronized String getLocalhost() { - if (myHost == null) { + if (cachedHostName == null) { try { - Process p = Runtime.getRuntime().exec("hostname"); - BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream())); - myHost = in.readLine(); - p.waitFor(); - if (p.exitValue() != 0) { - throw new RuntimeException("Command 'hostname' failed: exit("+p.exitValue()+")"); - } + cachedHostName = getPreferredAddress().canonicalHostName; } catch (Exception e) { - throw new RuntimeException("Failed when executing command 'hostname'", e); + throw new RuntimeException("Failed to find a preferred hostname", e); } } - return myHost; + return cachedHostName; + } + + private static Address getPreferredAddress() throws Exception { + List<Address> addresses = getReachableNetworkInterfaceAddresses(); + + // Prefer address matching the system hostname + String systemHostName = getSystemHostName(); + List<Address> systemAddresses = addresses.stream() + .filter(address -> Objects.equals(address.canonicalHostName, systemHostName)) + .collect(Collectors.toList()); + if (systemAddresses.size() >= 1) { + return systemAddresses.iterator().next(); + } + + // Otherwise, prefer non-local address. + List<Address> nonLocalAddresses = addresses.stream() + .filter(address -> !address.ipAddress.isAnyLocalAddress()) + .collect(Collectors.toList()); + if (nonLocalAddresses.size() >= 1) { + return nonLocalAddresses.iterator().next(); + } + + // Otherwise, pick a local address. + List<Address> localAddresses = addresses.stream() + .filter(address -> address.ipAddress.isAnyLocalAddress()) + .collect(Collectors.toList()); + if (localAddresses.size() >= 1) { + return localAddresses.iterator().next(); + } + + throw new RuntimeException("Failed to find any addresses on the network interfaces that resolves to a DNS name"); + } + + // public for testing purposes (all testing machines should have a hostname + public static String getSystemHostName() throws Exception { + Process process = Runtime.getRuntime().exec("hostname"); + BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream())); + String hostname = in.readLine(); + process.waitFor(); + if (process.exitValue() != 0) { + throw new RuntimeException("Command 'hostname' failed with exit code " + process.exitValue()); + } + + return hostname; + } + + private static class Address { + + public final InetAddress ipAddress; + public final String canonicalHostName; + + public Address(InetAddress ipAddress, String canonicalHostName) { + this.ipAddress = ipAddress; + this.canonicalHostName = canonicalHostName; + } + + } + + private static List<Address> getReachableNetworkInterfaceAddresses() throws SocketException { + List<Address> addresses = new ArrayList<>(); + + for (NetworkInterface networkInterface : Collections.list(NetworkInterface.getNetworkInterfaces())) { + for (InetAddress ipAddress : Collections.list(networkInterface.getInetAddresses())) { + String hostname = ipAddress.getCanonicalHostName(); + if (Objects.equals(hostname, ipAddress.getHostAddress())) { + // getCanonicalHostName() failed to get the fully qualified domain name + continue; + } + + try { + // ping says ~50ms on my Fedora Lenovo, but that seems a lot for pinging oneself - hakon + int timeoutMs = 100; + if ( ! ipAddress.isReachable(timeoutMs)) { + // The network interface may be down, ignore address + logger.log(Level.INFO, ipAddress.toString() + + " is unreachable w/" + timeoutMs + "ms timeout, ignoring address"); + continue; + } + } catch (IOException e) { + // Why would this be different from !isReachable ? + logger.log(Level.INFO, "Failed testing reachability of " + ipAddress + ", ignoring address", e); + continue; + } + + addresses.add(new Address(ipAddress, hostname)); + } + } + + return addresses; } } diff --git a/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java b/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java deleted file mode 100644 index 9d50c99d77c..00000000000 --- a/vespajlib/src/main/java/com/yahoo/net/LinuxInetAddress.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.net; - -import java.net.*; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Enumeration; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Utilities for returning localhost addresses on Linux. - * See http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4665037 on why this is necessary. - * - * @author bratseth - */ -public class LinuxInetAddress { - - /** - * Returns an InetAddress representing a resolvable localhost address. - * A non-loopback address is preferred if available. - * An address that resolves to a hostname is preferred among non-loopback addresses. - * IPv4 is preferred over IPv6 among resolving addresses. - * - * @return a localhost address - */ - // Note: Checking resolvability of ipV6 addresses takes a long time on some systems (over 5 seconds - // for some addresses on my mac). This method is written to minimize the number of resolution checks done - // and to defer ip6 checks until necessary. - public static InetAddress getLocalHost() { - InetAddress fallback = InetAddress.getLoopbackAddress(); - try { - fallback = InetAddress.getLocalHost(); - List<InetAddress> nonLoopback = - getAllLocalFromNetwork().stream().filter(a -> ! a.isLoopbackAddress()).collect(Collectors.toList()); - if (nonLoopback.isEmpty()) return fallback; - - // Invariant: We got all addresses without exception - - List<InetAddress> ipV4 = nonLoopback.stream().filter(a -> a instanceof Inet4Address).collect(Collectors.toList()); - for (InetAddress address : ipV4) - if (doesResolve(address)) - return address; - - // Invariant: There are no resolving ip4 addresses - - List<InetAddress> ipV6 = nonLoopback.stream().filter(a -> a instanceof Inet4Address).collect(Collectors.toList()); - for (InetAddress address : ipV6) - if (doesResolve(address)) - return address; - - // Invariant: There are no resolving ip6 addresses either - - if (! ipV4.isEmpty()) return ipV4.get(0); - return ipV6.get(0); - } catch (UnknownHostException e) { - return fallback; - } - } - - /** - * Returns all local addresses of this host. - * - * @return an array of the addresses of this - * @throws UnknownHostException if we cannot access the network - */ - public static InetAddress[] getAllLocal() throws UnknownHostException { - InetAddress[] localInetAddresses = InetAddress.getAllByName("127.0.0.1"); - if ( ! localInetAddresses[0].isLoopbackAddress()) return localInetAddresses; - return getAllLocalFromNetwork().toArray(new InetAddress[0]); - } - - /** - * Returns all local addresses of this host. - * - * @return a list of the addresses of this - * @throws UnknownHostException if we cannot access the network - */ - private static List<InetAddress> getAllLocalFromNetwork() throws UnknownHostException { - try { - List<InetAddress> addresses = new ArrayList<>(); - for (NetworkInterface networkInterface : Collections.list(NetworkInterface.getNetworkInterfaces())) - addresses.addAll(Collections.list(networkInterface.getInetAddresses())); - return addresses; - } - catch (SocketException ex) { - throw new UnknownHostException("127.0.0.1"); - } - } - - private static boolean doesResolve(InetAddress address) { - // The latter returns a name if resolvable to one and the host address otherwise - return ! address.getHostAddress().equals(address.getCanonicalHostName()); - } - -} diff --git a/vespajlib/src/test/java/com/yahoo/net/HostNameTestCase.java b/vespajlib/src/test/java/com/yahoo/net/HostNameTestCase.java index 98be9f0ef6f..ddcd05c5366 100644 --- a/vespajlib/src/test/java/com/yahoo/net/HostNameTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/net/HostNameTestCase.java @@ -9,8 +9,15 @@ import static org.junit.Assert.assertFalse; * @author lulf */ public class HostNameTestCase { + @Test public void testHostnameIsFound() { assertFalse(HostName.getLocalhost().isEmpty()); } + + @Test + public void testSystemHostnameIsFound() throws Exception { + assertFalse(HostName.getSystemHostName().isEmpty()); + } + } diff --git a/vespajlib/src/test/java/com/yahoo/net/LinuxInetAddressTestCase.java b/vespajlib/src/test/java/com/yahoo/net/LinuxInetAddressTestCase.java deleted file mode 100755 index 27b85a0864c..00000000000 --- a/vespajlib/src/test/java/com/yahoo/net/LinuxInetAddressTestCase.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.net;
-
-import java.net.UnknownHostException;
-import java.net.InetAddress;
-import java.net.Inet4Address;
-
-/**
- * @author Simon Thoresen
- */
-public class LinuxInetAddressTestCase extends junit.framework.TestCase {
-
- @SuppressWarnings("deprecation")
- public void testPreferIPv4() throws UnknownHostException {
- try {
- // This test only works if there is at least one inet address returned.
- InetAddress[] arr = LinuxInetAddress.getAllLocal();
- if (arr.length > 0) {
- // System.out.println("Got " + arr.length + " addresses.");
-
- // And it can only make sure it is preferred if there is at least one ip v4 address.
- boolean ipv4 = false;
- for (int i = 0; i < arr.length; ++i) {
- // System.out.println("Address " + i + " is an instance of " + arr[i].getClass() + ".");
- if (arr[i] instanceof Inet4Address) {
- ipv4 = true;
- }
- }
-
- // And the only thing we test is that an ip v4 address is preferred.
- if (ipv4) {
- InetAddress addr = LinuxInetAddress.getLocalHost();
- assertNotNull("IPv4 is prefered", addr instanceof Inet4Address);
- }
- }
- }
- catch (java.net.UnknownHostException e) {
- // We're on vpn or have no network
- }
- }
-
-}
diff --git a/vespalib/src/tests/alloc/alloc_test.cpp b/vespalib/src/tests/alloc/alloc_test.cpp index b16afbcc7a6..52ca4034611 100644 --- a/vespalib/src/tests/alloc/alloc_test.cpp +++ b/vespalib/src/tests/alloc/alloc_test.cpp @@ -48,51 +48,57 @@ void Test::testBasic() { { - Alloc h = HeapAllocFactory::create(100); + Alloc h = Alloc::allocHeap(100); EXPECT_EQUAL(100u, h.size()); - EXPECT_TRUE(h.get() != NULL); + EXPECT_TRUE(h.get() != nullptr); } { - EXPECT_EXCEPTION(AlignedHeapAllocFactory::create(100, 7), IllegalArgumentException, "AlignedHeapAllocFactory::create(100, 7) does not support 7 alignment"); - Alloc h = AlignedHeapAllocFactory::create(100, 1024); + EXPECT_EXCEPTION(Alloc::allocAlignedHeap(100, 7), IllegalArgumentException, "Alloc::allocAlignedHeap(100, 7) does not support 7 alignment"); + Alloc h = Alloc::allocAlignedHeap(100, 1024); EXPECT_EQUAL(100u, h.size()); - EXPECT_TRUE(h.get() != NULL); + EXPECT_TRUE(h.get() != nullptr); } { - Alloc h = MMapAllocFactory::create(100); + Alloc h = Alloc::allocMMap(100); EXPECT_EQUAL(100u, h.size()); - EXPECT_TRUE(h.get() != NULL); + EXPECT_TRUE(h.get() != nullptr); } { - Alloc a = HeapAllocFactory::create(100), b = HeapAllocFactory::create(200); + Alloc a = Alloc::allocHeap(100), b = Alloc::allocHeap(200); testSwap(a, b); } { - Alloc a = MMapAllocFactory::create(100), b = MMapAllocFactory::create(200); + Alloc a = Alloc::allocMMap(100), b = Alloc::allocMMap(200); testSwap(a, b); } { - Alloc a = AlignedHeapAllocFactory::create(100, 1024), b = AlignedHeapAllocFactory::create(200, 1024); + Alloc a = Alloc::allocAlignedHeap(100, 1024), b = Alloc::allocAlignedHeap(200, 1024); testSwap(a, b); } { - Alloc a = HeapAllocFactory::create(100); - Alloc b = MMapAllocFactory::create(200); + Alloc a = Alloc::allocHeap(100); + Alloc b = Alloc::allocMMap(200); testSwap(a, b); } + { + Alloc a = Alloc::allocHeap(100); + Alloc b = Alloc::allocHeap(100); + a = std::move(b); + EXPECT_TRUE(b.get() == nullptr); + } } void Test::testAlignedAllocation() { { - Alloc buf = AutoAllocFactory::create(10, MemoryAllocator::HUGEPAGE_SIZE, 1024); + Alloc buf = Alloc::alloc(10, MemoryAllocator::HUGEPAGE_SIZE, 1024); EXPECT_TRUE(reinterpret_cast<ptrdiff_t>(buf.get()) % 1024 == 0); } { // Mmapped pointers are page-aligned, but sanity test anyway. - Alloc buf = AutoAllocFactory::create(3000000, MemoryAllocator::HUGEPAGE_SIZE, 512); + Alloc buf = Alloc::alloc(3000000, MemoryAllocator::HUGEPAGE_SIZE, 512); EXPECT_TRUE(reinterpret_cast<ptrdiff_t>(buf.get()) % 512 == 0); } } diff --git a/vespalib/src/tests/alloc/allocate_and_core.cpp b/vespalib/src/tests/alloc/allocate_and_core.cpp index f0a0669eb42..63718de5ee4 100644 --- a/vespalib/src/tests/alloc/allocate_and_core.cpp +++ b/vespalib/src/tests/alloc/allocate_and_core.cpp @@ -6,9 +6,9 @@ using namespace vespalib::alloc; int main(int argc, char *argv[]) { (void) argc; (void) argv; - Alloc small(MMapAllocFactory::create(0x400000)); //4M + Alloc small(Alloc::allocMMap(0x400000)); //4M memset(small.get(), 0x55, small.size()); - Alloc large(MMapAllocFactory::create(0x4000000)); //640M + Alloc large(Alloc::allocMMap(0x4000000)); //640M memset(large.get(), 0x66, large.size()); assert(false); } diff --git a/vespalib/src/tests/exception_classes/mmap.cpp b/vespalib/src/tests/exception_classes/mmap.cpp index fe0a8fc4556..5068420eaff 100644 --- a/vespalib/src/tests/exception_classes/mmap.cpp +++ b/vespalib/src/tests/exception_classes/mmap.cpp @@ -15,7 +15,7 @@ int main(int argc, char *argv[]) { assert(setrlimit(RLIMIT_AS, &virtualLimit) == 0); std::vector<Alloc> mappings; for (size_t i(0); i < numBlocks; i++) { - mappings.emplace_back(MMapAllocFactory::create(blockSize)); + mappings.emplace_back(Alloc::allocMMap(blockSize)); memset(mappings.back().get(), 0xa5, mappings.back().size()); } return 0; diff --git a/vespalib/src/tests/tensor/dense_tensor_address_combiner/dense_tensor_address_combiner_test.cpp b/vespalib/src/tests/tensor/dense_tensor_address_combiner/dense_tensor_address_combiner_test.cpp index 1192469e006..37f95172251 100644 --- a/vespalib/src/tests/tensor/dense_tensor_address_combiner/dense_tensor_address_combiner_test.cpp +++ b/vespalib/src/tests/tensor/dense_tensor_address_combiner/dense_tensor_address_combiner_test.cpp @@ -5,32 +5,28 @@ #include <vespa/vespalib/test/insertion_operators.h> using namespace vespalib::tensor; -using DimensionsMeta = DenseTensor::DimensionsMeta; +using vespalib::eval::ValueType; -std::ostream & -operator<<(std::ostream &out, const DenseTensor::DimensionMeta &dimMeta) +ValueType +combine(const std::vector<ValueType::Dimension> &lhs, + const std::vector<ValueType::Dimension> &rhs) { - out << dimMeta.dimension() << "[" << dimMeta.size() << "]"; - return out; -} - -DimensionsMeta -combine(const DimensionsMeta &lhs, const DimensionsMeta &rhs) -{ - return DenseTensorAddressCombiner::combineDimensions(lhs, rhs); + return DenseTensorAddressCombiner::combineDimensions( + ValueType::tensor_type(lhs), + ValueType::tensor_type(rhs)); } TEST("require that dimensions can be combined") { - EXPECT_EQUAL(DimensionsMeta({{"a", 3}, {"b", 5}}), combine({{"a", 3}}, {{"b", 5}})); - EXPECT_EQUAL(DimensionsMeta({{"a", 3}, {"b", 5}}), combine({{"a", 3}, {"b", 5}}, {{"b", 5}})); - EXPECT_EQUAL(DimensionsMeta({{"a", 3}, {"b", 5}}), combine({{"a", 3}, {"b", 7}}, {{"b", 5}})); - EXPECT_EQUAL(DimensionsMeta({{"a", 3}, {"b", 11}, {"c", 5}, {"d", 7}, {"e", 17}}), + EXPECT_EQUAL(ValueType::tensor_type({{"a", 3}, {"b", 5}}), combine({{"a", 3}}, {{"b", 5}})); + EXPECT_EQUAL(ValueType::tensor_type({{"a", 3}, {"b", 5}}), combine({{"a", 3}, {"b", 5}}, {{"b", 5}})); + EXPECT_EQUAL(ValueType::tensor_type({{"a", 3}, {"b", 5}}), combine({{"a", 3}, {"b", 7}}, {{"b", 5}})); + EXPECT_EQUAL(ValueType::tensor_type({{"a", 3}, {"b", 11}, {"c", 5}, {"d", 7}, {"e", 17}}), combine({{"a", 3}, {"c", 5}, {"d", 7}}, {{"b", 11}, {"c", 13}, {"e", 17}})); - EXPECT_EQUAL(DimensionsMeta({{"a", 3}, {"b", 11}, {"c", 5}, {"d", 7}, {"e", 17}}), - combine({{"b", 11}, {"c", 13}, {"e", 17}}, - {{"a", 3}, {"c", 5}, {"d", 7}})); + EXPECT_EQUAL(ValueType::tensor_type({{"a", 3}, {"b", 11}, {"c", 5}, {"d", 7}, {"e", 17}}), + combine({{"b", 11}, {"c", 13}, {"e", 17}}, + {{"a", 3}, {"c", 5}, {"d", 7}})); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp b/vespalib/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp index 595b3743625..5036f247db3 100644 --- a/vespalib/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp +++ b/vespalib/src/tests/tensor/dense_tensor_builder/dense_tensor_builder_test.cpp @@ -9,14 +9,15 @@ using namespace vespalib::tensor; using vespalib::IllegalArgumentException; using Builder = DenseTensorBuilder; using vespalib::eval::TensorSpec; +using vespalib::eval::ValueType; void -assertTensor(const DenseTensor::DimensionsMeta &expDims, +assertTensor(const std::vector<ValueType::Dimension> &expDims, const DenseTensor::Cells &expCells, const Tensor &tensor) { const DenseTensor &realTensor = dynamic_cast<const DenseTensor &>(tensor); - EXPECT_EQUAL(expDims, realTensor.dimensionsMeta()); + EXPECT_EQUAL(ValueType::tensor_type(expDims), realTensor.type()); EXPECT_EQUAL(expCells, realTensor.cells()); } diff --git a/vespalib/src/tests/tensor/sparse_tensor_builder/sparse_tensor_builder_test.cpp b/vespalib/src/tests/tensor/sparse_tensor_builder/sparse_tensor_builder_test.cpp index 0d157012a90..d1ad41e8a7e 100644 --- a/vespalib/src/tests/tensor/sparse_tensor_builder/sparse_tensor_builder_test.cpp +++ b/vespalib/src/tests/tensor/sparse_tensor_builder/sparse_tensor_builder_test.cpp @@ -6,21 +6,22 @@ using namespace vespalib::tensor; using vespalib::eval::TensorSpec; +using vespalib::eval::ValueType; void assertCellValue(double expValue, const TensorAddress &address, - const TensorDimensions &dimensions, + const ValueType &type, const SparseTensor::Cells &cells) { SparseTensorAddressBuilder addressBuilder; - auto dimsItr = dimensions.cbegin(); - auto dimsItrEnd = dimensions.cend(); + auto dimsItr = type.dimensions().cbegin(); + auto dimsItrEnd = type.dimensions().cend(); for (const auto &element : address.elements()) { - while ((dimsItr < dimsItrEnd) && (*dimsItr < element.dimension())) { + while ((dimsItr < dimsItrEnd) && (dimsItr->name < element.dimension())) { addressBuilder.add(""); ++dimsItr; } - assert((dimsItr != dimsItrEnd) && (*dimsItr == element.dimension())); + assert((dimsItr != dimsItrEnd) && (dimsItr->name == element.dimension())); addressBuilder.add(element.label()); ++dimsItr; } @@ -53,13 +54,13 @@ TEST("require that tensor can be constructed") { Tensor::UP tensor = buildTensor(); const SparseTensor &sparseTensor = dynamic_cast<const SparseTensor &>(*tensor); - const TensorDimensions &dimensions = sparseTensor.dimensions(); + const ValueType &type = sparseTensor.type(); const SparseTensor::Cells &cells = sparseTensor.cells(); EXPECT_EQUAL(2u, cells.size()); assertCellValue(10, TensorAddress({{"a","1"},{"b","2"}}), - dimensions, cells); + type, cells); assertCellValue(20, TensorAddress({{"c","3"},{"d","4"}}), - dimensions, cells); + type, cells); } TEST("require that tensor can be converted to tensor spec") @@ -85,11 +86,11 @@ TEST("require that dimensions are extracted") add_label(builder.define_dimension("c"), "4").add_cell(20); Tensor::UP tensor = builder.build(); const SparseTensor &sparseTensor = dynamic_cast<const SparseTensor &>(*tensor); - const TensorDimensions &dims = sparseTensor.dimensions(); + const auto &dims = sparseTensor.type().dimensions(); EXPECT_EQUAL(3u, dims.size()); - EXPECT_EQUAL("a", dims[0]); - EXPECT_EQUAL("b", dims[1]); - EXPECT_EQUAL("c", dims[2]); + EXPECT_EQUAL("a", dims[0].name); + EXPECT_EQUAL("b", dims[1].name); + EXPECT_EQUAL("c", dims[2].name); EXPECT_EQUAL("tensor(a{},b{},c{})", sparseTensor.getType().to_spec()); } diff --git a/vespalib/src/vespa/vespalib/data/memorydatastore.cpp b/vespalib/src/vespa/vespalib/data/memorydatastore.cpp index 791ea0cea50..824a8adf1d6 100644 --- a/vespalib/src/vespa/vespalib/data/memorydatastore.cpp +++ b/vespalib/src/vespa/vespalib/data/memorydatastore.cpp @@ -41,7 +41,7 @@ MemoryDataStore::push_back(const void * data, const size_t sz) VariableSizeVector::VariableSizeVector(size_t initialSize) : _vector(), - _store(DefaultAlloc::create(initialSize)) + _store(Alloc::alloc(initialSize)) { } diff --git a/vespalib/src/vespa/vespalib/data/memorydatastore.h b/vespalib/src/vespa/vespalib/data/memorydatastore.h index b61fa3665ad..dd5809c1423 100644 --- a/vespalib/src/vespa/vespalib/data/memorydatastore.h +++ b/vespalib/src/vespa/vespalib/data/memorydatastore.h @@ -23,7 +23,7 @@ public: private: void * _data; }; - MemoryDataStore(alloc::Alloc && initialAlloc=DefaultAlloc::create(256), Lock * lock=nullptr); + MemoryDataStore(alloc::Alloc && initialAlloc=alloc::Alloc::alloc(256), Lock * lock=nullptr); MemoryDataStore(const MemoryDataStore &) = delete; MemoryDataStore & operator = (const MemoryDataStore &) = delete; ~MemoryDataStore(); diff --git a/vespalib/src/vespa/vespalib/eval/llvm/CMakeLists.txt b/vespalib/src/vespa/vespalib/eval/llvm/CMakeLists.txt index 78462a68282..238dceb4026 100644 --- a/vespalib/src/vespa/vespalib/eval/llvm/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/eval/llvm/CMakeLists.txt @@ -7,5 +7,6 @@ vespa_add_library(vespalib_vespalib_eval_llvm llvm_wrapper.cpp INSTALL lib64 DEPENDS + vespalib LLVM-3.4 ) diff --git a/vespalib/src/vespa/vespalib/tensor/CMakeLists.txt b/vespalib/src/vespa/vespalib/tensor/CMakeLists.txt index 00a582d99a7..7ed5e4d60d5 100644 --- a/vespalib/src/vespa/vespalib/tensor/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/tensor/CMakeLists.txt @@ -12,4 +12,5 @@ vespa_add_library(vespalib_vespalib_tensor $<TARGET_OBJECTS:vespalib_vespalib_tensor_serialization> INSTALL lib64 DEPENDS + vespalib ) diff --git a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.cpp b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.cpp index 5e81e9cb05d..b8cb0838fee 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.cpp +++ b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.cpp @@ -20,28 +20,28 @@ namespace tensor { namespace { string -dimensionsMetaAsString(const DenseTensor::DimensionsMeta &dimensionsMeta) +dimensionsAsString(const eval::ValueType &type) { std::ostringstream oss; bool first = true; oss << "["; - for (const auto &dimMeta : dimensionsMeta) { + for (const auto &dim : type.dimensions()) { if (!first) { oss << ","; } first = false; - oss << dimMeta; + oss << dim.name << ":" << dim.size; } oss << "]"; return oss.str(); } size_t -calcCellsSize(const DenseTensor::DimensionsMeta &dimensionsMeta) +calcCellsSize(const eval::ValueType &type) { size_t cellsSize = 1; - for (const auto &dimMeta : dimensionsMeta) { - cellsSize *= dimMeta.size(); + for (const auto &dim : type.dimensions()) { + cellsSize *= dim.size; } return cellsSize; } @@ -50,7 +50,7 @@ calcCellsSize(const DenseTensor::DimensionsMeta &dimensionsMeta) void checkCellsSize(const DenseTensor &arg) { - auto cellsSize = calcCellsSize(arg.dimensionsMeta()); + auto cellsSize = calcCellsSize(arg.type()); if (arg.cells().size() != cellsSize) { throw IllegalStateException(make_string("wrong cell size, " "expected=%zu, " @@ -64,14 +64,14 @@ void checkDimensions(const DenseTensor &lhs, const DenseTensor &rhs, vespalib::stringref operation) { - if (lhs.dimensionsMeta() != rhs.dimensionsMeta()) { - throw IllegalStateException(make_string("mismatching dimensions meta for " + if (lhs.type() != rhs.type()) { + throw IllegalStateException(make_string("mismatching dimensions for " "dense tensor %s, " - "lhs dimensions meta = '%s', " - "rhs dimensions meta = '%s'", + "lhs dimensions = '%s', " + "rhs dimensions = '%s'", operation.c_str(), - dimensionsMetaAsString(lhs.dimensionsMeta()).c_str(), - dimensionsMetaAsString(rhs.dimensionsMeta()).c_str())); + dimensionsAsString(lhs.type()).c_str(), + dimensionsAsString(rhs.type()).c_str())); } checkCellsSize(lhs); checkCellsSize(rhs); @@ -87,7 +87,7 @@ checkDimensions(const DenseTensor &lhs, const DenseTensor &rhs, template <typename Function> Tensor::UP joinDenseTensors(const DenseTensor &lhs, const DenseTensor &rhs, - Function &&func) + Function &&func) { DenseTensor::Cells cells; cells.reserve(lhs.cells().size()); @@ -97,42 +97,10 @@ joinDenseTensors(const DenseTensor &lhs, const DenseTensor &rhs, ++rhsCellItr; } assert(rhsCellItr == rhs.cells().cend()); - return std::make_unique<DenseTensor>(lhs.dimensionsMeta(), + return std::make_unique<DenseTensor>(lhs.type(), std::move(cells)); } -/* - * Join the cells of two tensors, where the rhs values are treated as negated values. - * The given function is used to calculate the resulting cell value for overlapping cells. - */ -template <typename Function> -Tensor::UP -joinDenseTensorsNegated(const DenseTensor &lhs, - const DenseTensor &rhs, - Function &&func) -{ - DenseTensor::Cells cells; - cells.reserve(lhs.cells().size()); - auto rhsCellItr = rhs.cells().cbegin(); - for (const auto &lhsCell : lhs.cells()) { - cells.push_back(func(lhsCell, - *rhsCellItr)); - ++rhsCellItr; - } - assert(rhsCellItr == rhs.cells().cend()); - return std::make_unique<DenseTensor>(lhs.dimensionsMeta(), - std::move(cells)); -} - -std::vector<vespalib::string> -getDimensions(const DenseTensor &tensor) -{ - std::vector<vespalib::string> dimensions; - for (const auto &dimMeta : tensor.dimensionsMeta()) { - dimensions.emplace_back(dimMeta.dimension()); - } - return dimensions; -} - } @@ -142,7 +110,7 @@ DenseTensor::CellsIterator::next() ++_cellIdx; if (valid()) { for (int64_t i = (_address.size() - 1); i >= 0; --i) { - _address[i] = (_address[i] + 1) % _dimensionsMeta[i].size(); + _address[i] = (_address[i] + 1) % _type.dimensions()[i].size; if (_address[i] != 0) { // Outer dimension labels can only be increased when this label wraps around. break; @@ -152,31 +120,31 @@ DenseTensor::CellsIterator::next() } DenseTensor::DenseTensor() - : _dimensionsMeta(), + : _type(eval::ValueType::double_type()), _cells(1) { } -DenseTensor::DenseTensor(const DimensionsMeta &dimensionsMeta_in, +DenseTensor::DenseTensor(const eval::ValueType &type_in, const Cells &cells_in) - : _dimensionsMeta(dimensionsMeta_in), + : _type(type_in), _cells(cells_in) { checkCellsSize(*this); } -DenseTensor::DenseTensor(const DimensionsMeta &dimensionsMeta_in, +DenseTensor::DenseTensor(const eval::ValueType &type_in, Cells &&cells_in) - : _dimensionsMeta(dimensionsMeta_in), + : _type(type_in), _cells(std::move(cells_in)) { checkCellsSize(*this); } -DenseTensor::DenseTensor(DimensionsMeta &&dimensionsMeta_in, +DenseTensor::DenseTensor(eval::ValueType &&type_in, Cells &&cells_in) - : _dimensionsMeta(std::move(dimensionsMeta_in)), + : _type(std::move(type_in)), _cells(std::move(cells_in)) { checkCellsSize(*this); @@ -185,23 +153,14 @@ DenseTensor::DenseTensor(DimensionsMeta &&dimensionsMeta_in, bool DenseTensor::operator==(const DenseTensor &rhs) const { - return (_dimensionsMeta == rhs._dimensionsMeta) && + return (_type == rhs._type) && (_cells == rhs._cells); } eval::ValueType DenseTensor::getType() const { - if (_dimensionsMeta.empty()) { - return eval::ValueType::double_type(); - } - std::vector<eval::ValueType::Dimension> dimensions; - dimensions.reserve(_dimensionsMeta.size()); - for (const auto &dimensionMeta : _dimensionsMeta) { - dimensions.emplace_back(dimensionMeta.dimension(), - dimensionMeta.size()); - } - return eval::ValueType::tensor_type(dimensions); + return _type; } double @@ -296,7 +255,7 @@ DenseTensor::apply(const CellFunction &func) const ++itr; } assert(itr == newCells.end()); - return std::make_unique<DenseTensor>(_dimensionsMeta, + return std::make_unique<DenseTensor>(_type, std::move(newCells)); } @@ -329,7 +288,7 @@ DenseTensor::toString() const Tensor::UP DenseTensor::clone() const { - return std::make_unique<DenseTensor>(_dimensionsMeta, _cells); + return std::make_unique<DenseTensor>(_type, _cells); } namespace { @@ -338,8 +297,8 @@ void buildAddress(const DenseTensor::CellsIterator &itr, TensorSpec::Address &address) { auto addressItr = itr.address().begin(); - for (const auto &dim : itr.dimensions()) { - address.emplace(std::make_pair(dim.dimension(), TensorSpec::Label(*addressItr++))); + for (const auto &dim : itr.type().dimensions()) { + address.emplace(std::make_pair(dim.name, TensorSpec::Label(*addressItr++))); } assert(addressItr == itr.address().end()); } @@ -351,7 +310,7 @@ DenseTensor::toSpec() const { TensorSpec result(getType().to_spec()); TensorSpec::Address address; - for (CellsIterator itr(_dimensionsMeta, _cells); itr.valid(); itr.next()) { + for (CellsIterator itr(_type, _cells); itr.valid(); itr.next()) { buildAddress(itr, address); result.add(address, itr.cell()); address.clear(); @@ -365,11 +324,11 @@ DenseTensor::print(std::ostream &out) const // TODO (geirst): print on common format. out << "[ "; bool first = true; - for (const auto &dimMeta : _dimensionsMeta) { + for (const auto &dim : _type.dimensions()) { if (!first) { out << ", "; } - out << dimMeta; + out << dim.name << ":" << dim.size; first = false; } out << " ] { "; @@ -387,16 +346,16 @@ DenseTensor::print(std::ostream &out) const void DenseTensor::accept(TensorVisitor &visitor) const { - DenseTensor::CellsIterator iterator(_dimensionsMeta, _cells); + DenseTensor::CellsIterator iterator(_type, _cells); TensorAddressBuilder addressBuilder; TensorAddress address; vespalib::string label; while (iterator.valid()) { addressBuilder.clear(); auto rawIndex = iterator.address().begin(); - for (const auto &dimension : _dimensionsMeta) { + for (const auto &dimension : _type.dimensions()) { label = vespalib::make_string("%zu", *rawIndex); - addressBuilder.add(dimension.dimension(), label); + addressBuilder.add(dimension.name, label); ++rawIndex; } address = addressBuilder.build(); @@ -405,13 +364,6 @@ DenseTensor::accept(TensorVisitor &visitor) const } } -std::ostream & -operator<<(std::ostream &out, const DenseTensor::DimensionMeta &value) -{ - out << value.dimension() << ":" << value.size(); - return out; -} - Tensor::UP DenseTensor::apply(const eval::BinaryOperation &op, const Tensor &arg) const { @@ -429,7 +381,7 @@ DenseTensor::reduce(const eval::BinaryOperation &op, const std::vector<vespalib::string> &dimensions) const { return dense::reduce(*this, - (dimensions.empty() ? getDimensions(*this) : dimensions), + (dimensions.empty() ? _type.dimension_names() : dimensions), [&op](double lhsValue, double rhsValue) { return op.eval(lhsValue, rhsValue); }); } diff --git a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.h b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.h index 104fddeee7e..0a253f398b2 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.h +++ b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor.h @@ -4,6 +4,7 @@ #include <vespa/vespalib/tensor/tensor.h> #include <vespa/vespalib/tensor/types.h> +#include <vespa/vespalib/eval/value_type.h> namespace vespalib { namespace tensor { @@ -18,78 +19,46 @@ public: typedef std::unique_ptr<DenseTensor> UP; using Cells = std::vector<double>; - class DimensionMeta - { - vespalib::string _dimension; - size_t _size; - - public: - DimensionMeta(const vespalib::string & dimension_in, size_t size_in) - : _dimension(dimension_in), - _size(size_in) - { - } - - const vespalib::string &dimension() const { return _dimension; } - size_t size() const { return _size; } - - bool operator==(const DimensionMeta &rhs) const { - return (_dimension == rhs._dimension) && - (_size == rhs._size); - } - bool operator!=(const DimensionMeta &rhs) const { - return !(*this == rhs); - } - bool operator<(const DimensionMeta &rhs) const { - if (_dimension == rhs._dimension) { - return _size < rhs._size; - } - return _dimension < rhs._dimension; - } - }; - - using DimensionsMeta = std::vector<DimensionMeta>; - class CellsIterator { private: - const DimensionsMeta &_dimensionsMeta; + const eval::ValueType &_type; const Cells &_cells; size_t _cellIdx; std::vector<size_t> _address; public: - CellsIterator(const DimensionsMeta &dimensionsMeta, + CellsIterator(const eval::ValueType &type_in, const Cells &cells) - : _dimensionsMeta(dimensionsMeta), + : _type(type_in), _cells(cells), _cellIdx(0), - _address(dimensionsMeta.size(), 0) + _address(type_in.dimensions().size(), 0) {} bool valid() const { return _cellIdx < _cells.size(); } void next(); double cell() const { return _cells[_cellIdx]; } const std::vector<size_t> &address() const { return _address; } - const DimensionsMeta &dimensions() const { return _dimensionsMeta; } + const eval::ValueType &type() const { return _type; } }; private: - DimensionsMeta _dimensionsMeta; + eval::ValueType _type; Cells _cells; public: DenseTensor(); - DenseTensor(const DimensionsMeta &dimensionsMeta_in, + DenseTensor(const eval::ValueType &type_in, const Cells &cells_in); - DenseTensor(const DimensionsMeta &dimensionsMeta_in, + DenseTensor(const eval::ValueType &type_in, Cells &&cells_in); - DenseTensor(DimensionsMeta &&dimensionsMeta_in, + DenseTensor(eval::ValueType &&type_in, Cells &&cells_in); - const DimensionsMeta &dimensionsMeta() const { return _dimensionsMeta; } + const eval::ValueType &type() const { return _type; } const Cells &cells() const { return _cells; } bool operator==(const DenseTensor &rhs) const; - CellsIterator cellsIterator() const { return CellsIterator(_dimensionsMeta, _cells); } + CellsIterator cellsIterator() const { return CellsIterator(_type, _cells); } virtual eval::ValueType getType() const override; virtual double sum() const override; @@ -114,7 +83,5 @@ public: virtual void accept(TensorVisitor &visitor) const override; }; -std::ostream &operator<<(std::ostream &out, const DenseTensor::DimensionMeta &value); - } // namespace vespalib::tensor } // namespace vespalib diff --git a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_address_combiner.cpp b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_address_combiner.cpp index 88fe86ca9e6..1a3780b8f66 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_address_combiner.cpp +++ b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_address_combiner.cpp @@ -9,8 +9,6 @@ namespace vespalib { namespace tensor { using Address = DenseTensorAddressCombiner::Address; -using DimensionMeta = DenseTensor::DimensionMeta; -using DimensionsMeta = DenseTensorAddressCombiner::DimensionsMeta; namespace { @@ -35,19 +33,19 @@ public: } -DenseTensorAddressCombiner::DenseTensorAddressCombiner(const DimensionsMeta &lhs, - const DimensionsMeta &rhs) +DenseTensorAddressCombiner::DenseTensorAddressCombiner(const eval::ValueType &lhs, + const eval::ValueType &rhs) : _ops(), _combinedAddress() { - auto rhsItr = rhs.cbegin(); - auto rhsItrEnd = rhs.cend(); - for (const auto &lhsDim : lhs) { - while ((rhsItr != rhsItrEnd) && (rhsItr->dimension() < lhsDim.dimension())) { + auto rhsItr = rhs.dimensions().cbegin(); + auto rhsItrEnd = rhs.dimensions().cend(); + for (const auto &lhsDim : lhs.dimensions()) { + while ((rhsItr != rhsItrEnd) && (rhsItr->name < lhsDim.name)) { _ops.push_back(AddressOp::RHS); ++rhsItr; } - if ((rhsItr != rhsItrEnd) && (rhsItr->dimension() == lhsDim.dimension())) { + if ((rhsItr != rhsItrEnd) && (rhsItr->name == lhsDim.name)) { _ops.push_back(AddressOp::BOTH); ++rhsItr; } else { @@ -89,31 +87,36 @@ DenseTensorAddressCombiner::combine(const CellsIterator &lhsItr, return true; } -DimensionsMeta -DenseTensorAddressCombiner::combineDimensions(const DimensionsMeta &lhs, const DimensionsMeta &rhs) +eval::ValueType +DenseTensorAddressCombiner::combineDimensions(const eval::ValueType &lhs, + const eval::ValueType &rhs) { // NOTE: both lhs and rhs are sorted according to dimension names. - DimensionsMeta result; - auto lhsItr = lhs.cbegin(); - auto rhsItr = rhs.cbegin(); - while (lhsItr != lhs.end() && rhsItr != rhs.end()) { - if (lhsItr->dimension() == rhsItr->dimension()) { - result.emplace_back(DimensionMeta(lhsItr->dimension(), std::min(lhsItr->size(), rhsItr->size()))); + std::vector<eval::ValueType::Dimension> result; + auto lhsItr = lhs.dimensions().cbegin(); + auto rhsItr = rhs.dimensions().cbegin(); + while (lhsItr != lhs.dimensions().end() && + rhsItr != rhs.dimensions().end()) { + if (lhsItr->name == rhsItr->name) { + result.emplace_back(lhsItr->name, + std::min(lhsItr->size, rhsItr->size)); ++lhsItr; ++rhsItr; - } else if (lhsItr->dimension() < rhsItr->dimension()) { + } else if (lhsItr->name < rhsItr->name) { result.emplace_back(*lhsItr++); } else { result.emplace_back(*rhsItr++); } } - while (lhsItr != lhs.end()) { + while (lhsItr != lhs.dimensions().end()) { result.emplace_back(*lhsItr++); } - while (rhsItr != rhs.end()) { + while (rhsItr != rhs.dimensions().end()) { result.emplace_back(*rhsItr++); } - return result; + return (result.empty() ? + eval::ValueType::double_type() : + eval::ValueType::tensor_type(result)); } } // namespace vespalib::tensor diff --git a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_address_combiner.h b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_address_combiner.h index 2c7f9e61223..89168e038bc 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_address_combiner.h +++ b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_address_combiner.h @@ -16,7 +16,6 @@ class DenseTensorAddressCombiner { public: using Address = std::vector<size_t>; - using DimensionsMeta = DenseTensor::DimensionsMeta; private: enum class AddressOp { @@ -31,14 +30,14 @@ private: Address _combinedAddress; public: - DenseTensorAddressCombiner(const DimensionsMeta &lhs, - const DimensionsMeta &rhs); + DenseTensorAddressCombiner(const eval::ValueType &lhs, + const eval::ValueType &rhs); bool combine(const CellsIterator &lhsItr, const CellsIterator &rhsItr); const Address &address() const { return _combinedAddress; } - static DimensionsMeta combineDimensions(const DimensionsMeta &lhs, const DimensionsMeta &rhs); + static eval::ValueType combineDimensions(const eval::ValueType &lhs, const eval::ValueType &rhs); }; diff --git a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_apply.hpp b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_apply.hpp index 3168089b941..270539f72d8 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_apply.hpp +++ b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_apply.hpp @@ -14,8 +14,8 @@ template <typename Function> std::unique_ptr<Tensor> apply(const DenseTensor &lhs, const DenseTensor &rhs, Function &&func) { - DenseTensorAddressCombiner combiner(lhs.dimensionsMeta(), rhs.dimensionsMeta()); - DirectDenseTensorBuilder builder(DenseTensorAddressCombiner::combineDimensions(lhs.dimensionsMeta(), rhs.dimensionsMeta())); + DenseTensorAddressCombiner combiner(lhs.type(), rhs.type()); + DirectDenseTensorBuilder builder(DenseTensorAddressCombiner::combineDimensions(lhs.type(), rhs.type())); for (DenseTensor::CellsIterator lhsItr = lhs.cellsIterator(); lhsItr.valid(); lhsItr.next()) { for (DenseTensor::CellsIterator rhsItr = rhs.cellsIterator(); rhsItr.valid(); rhsItr.next()) { bool combineSuccess = combiner.combine(lhsItr, rhsItr); diff --git a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_builder.cpp b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_builder.cpp index ab0404f320b..d5e6feb2135 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_builder.cpp +++ b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_builder.cpp @@ -13,8 +13,6 @@ using vespalib::make_string; namespace vespalib { namespace tensor { -using DimensionMeta = DenseTensor::DimensionMeta; - namespace { constexpr size_t UNDEFINED_LABEL = std::numeric_limits<size_t>::max(); @@ -39,14 +37,21 @@ validateLabelNotSpecified(size_t oldLabel, const vespalib::string &dimension) } } +eval::ValueType +makeValueType(const std::vector<eval::ValueType::Dimension> &&dimensions) { + return (dimensions.empty() ? + eval::ValueType::double_type() : + eval::ValueType::tensor_type(std::move(dimensions))); +} + } void DenseTensorBuilder::allocateCellsStorage() { size_t cellsSize = 1; - for (const auto &dimensionMeta : _dimensionsMeta) { - cellsSize *= dimensionMeta.size(); + for (const auto &dimension : _dimensions) { + cellsSize *= dimension.size; } _cells.resize(cellsSize, 0); } @@ -55,13 +60,14 @@ DenseTensorBuilder::allocateCellsStorage() void DenseTensorBuilder::sortDimensions() { - std::sort(_dimensionsMeta.begin(), _dimensionsMeta.end(), - [](const DimensionMeta &lhs, const DimensionMeta &rhs) - { return lhs.dimension() < rhs.dimension(); }); - _dimensionsMapping.resize(_dimensionsMeta.size()); + std::sort(_dimensions.begin(), _dimensions.end(), + [](const eval::ValueType::Dimension &lhs, + const eval::ValueType::Dimension &rhs) + { return lhs.name < rhs.name; }); + _dimensionsMapping.resize(_dimensions.size()); Dimension dim = 0; - for (const auto &dimension : _dimensionsMeta) { - auto itr = _dimensionsEnum.find(dimension.dimension()); + for (const auto &dimension : _dimensions) { + auto itr = _dimensionsEnum.find(dimension.name); assert(itr != _dimensionsEnum.end()); _dimensionsMapping[itr->second] = dim; ++dim; @@ -75,14 +81,14 @@ DenseTensorBuilder::calculateCellAddress() size_t multiplier = 1; for (int64_t i = (_addressBuilder.size() - 1); i >= 0; --i) { const size_t label = _addressBuilder[i]; - const auto &dimMeta = _dimensionsMeta[i]; + const auto &dim = _dimensions[i]; if (label == UNDEFINED_LABEL) { throw IllegalArgumentException(make_string("Label for dimension '%s' is undefined. " "Expected a value in the range [0, %zu>", - dimMeta.dimension().c_str(), dimMeta.size())); + dim.name.c_str(), dim.size)); } result += (label * multiplier); - multiplier *= dimMeta.size(); + multiplier *= dim.size; _addressBuilder[i] = UNDEFINED_LABEL; } return result; @@ -90,7 +96,7 @@ DenseTensorBuilder::calculateCellAddress() DenseTensorBuilder::DenseTensorBuilder() : _dimensionsEnum(), - _dimensionsMeta(), + _dimensions(), _cells(), _addressBuilder(), _dimensionsMapping() @@ -108,9 +114,9 @@ DenseTensorBuilder::defineDimension(const vespalib::string &dimension, assert(_cells.empty()); Dimension result = _dimensionsEnum.size(); _dimensionsEnum.insert(std::make_pair(dimension, result)); - _dimensionsMeta.emplace_back(dimension, dimensionSize); + _dimensions.emplace_back(dimension, dimensionSize); _addressBuilder.push_back(UNDEFINED_LABEL); - assert(_dimensionsMeta.size() == (result + 1)); + assert(_dimensions.size() == (result + 1)); assert(_addressBuilder.size() == (result + 1)); return result; } @@ -122,13 +128,13 @@ DenseTensorBuilder::addLabel(Dimension dimension, size_t label) sortDimensions(); allocateCellsStorage(); } - assert(dimension < _dimensionsMeta.size()); + assert(dimension < _dimensions.size()); assert(dimension < _addressBuilder.size()); Dimension mappedDimension = _dimensionsMapping[dimension]; - const auto &dimMeta = _dimensionsMeta[mappedDimension]; - validateLabelInRange(label, dimMeta.size(), dimMeta.dimension()); + const auto &dim = _dimensions[mappedDimension]; + validateLabelInRange(label, dim.size, dim.name); validateLabelNotSpecified(_addressBuilder[mappedDimension], - dimMeta.dimension()); + dim.name); _addressBuilder[mappedDimension] = label; return *this; } @@ -152,10 +158,10 @@ DenseTensorBuilder::build() if (_cells.empty()) { allocateCellsStorage(); } - Tensor::UP result = std::make_unique<DenseTensor>(std::move(_dimensionsMeta), - std::move(_cells)); + Tensor::UP result = std::make_unique<DenseTensor>(makeValueType(std::move(_dimensions)), + std::move(_cells)); _dimensionsEnum.clear(); - _dimensionsMeta.clear(); + _dimensions.clear(); DenseTensor::Cells().swap(_cells); _addressBuilder.clear(); _dimensionsMapping.clear(); diff --git a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_builder.h b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_builder.h index 1533ff3ba61..31e3b7cf451 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_builder.h +++ b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_builder.h @@ -19,7 +19,7 @@ public: private: vespalib::hash_map<vespalib::string, size_t> _dimensionsEnum; - DenseTensor::DimensionsMeta _dimensionsMeta; + std::vector<eval::ValueType::Dimension> _dimensions; DenseTensor::Cells _cells; std::vector<size_t> _addressBuilder; std::vector<Dimension> _dimensionsMapping; diff --git a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_reduce.hpp b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_reduce.hpp index e2af832f068..b072b7ef206 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_reduce.hpp +++ b/vespalib/src/vespa/vespalib/tensor/dense/dense_tensor_reduce.hpp @@ -8,31 +8,15 @@ namespace tensor { namespace dense { using Cells = DenseTensor::Cells; -using DimensionsMeta = DenseTensor::DimensionsMeta; namespace { -DimensionsMeta -removeDimension(const DimensionsMeta &dimensionsMeta, - const string &dimensionToRemove) -{ - DimensionsMeta result = dimensionsMeta; - auto itr = std::lower_bound(result.begin(), result.end(), dimensionToRemove, - [](const auto &dimMeta, const auto &dimension_in) { - return dimMeta.dimension() < dimension_in; - }); - if ((itr != result.end()) && (itr->dimension() == dimensionToRemove)) { - result.erase(itr); - } - return result; -} - size_t -calcCellsSize(const DimensionsMeta &dimensionsMeta) +calcCellsSize(const eval::ValueType &type) { size_t cellsSize = 1; - for (const auto &dimMeta : dimensionsMeta) { - cellsSize *= dimMeta.size(); + for (const auto &dim : type.dimensions()) { + cellsSize *= dim.size; } return cellsSize; } @@ -41,41 +25,42 @@ calcCellsSize(const DimensionsMeta &dimensionsMeta) class DimensionReducer { private: - DimensionsMeta _dimensionsResult; + eval::ValueType _type; Cells _cellsResult; size_t _innerDimSize; size_t _sumDimSize; size_t _outerDimSize; - void setup(const DimensionsMeta &dimensions, + void setup(const eval::ValueType &oldType, const vespalib::string &dimensionToRemove) { - auto itr = std::lower_bound(dimensions.cbegin(), dimensions.cend(), dimensionToRemove, - [](const auto &dimMeta, const auto &dimension) { - return dimMeta.dimension() < dimension; - }); - if ((itr != dimensions.end()) && (itr->dimension() == dimensionToRemove)) { - for (auto outerItr = dimensions.cbegin(); outerItr != itr; ++outerItr) { - _outerDimSize *= outerItr->size(); + auto itr = std::lower_bound(oldType.dimensions().cbegin(), + oldType.dimensions().cend(), + dimensionToRemove, + [](const auto &dim, const auto &dimension) + { return dim.name < dimension; }); + if ((itr != oldType.dimensions().end()) && (itr->name == dimensionToRemove)) { + for (auto outerItr = oldType.dimensions().cbegin(); outerItr != itr; ++outerItr) { + _outerDimSize *= outerItr->size; } - _sumDimSize = itr->size(); - for (++itr; itr != dimensions.cend(); ++itr) { - _innerDimSize *= itr->size(); + _sumDimSize = itr->size; + for (++itr; itr != oldType.dimensions().cend(); ++itr) { + _innerDimSize *= itr->size; } } else { - _outerDimSize = calcCellsSize(dimensions); + _outerDimSize = calcCellsSize(oldType); } } public: - DimensionReducer(const DimensionsMeta &dimensions, + DimensionReducer(const eval::ValueType &oldType, const string &dimensionToRemove) - : _dimensionsResult(removeDimension(dimensions, dimensionToRemove)), - _cellsResult(calcCellsSize(_dimensionsResult)), + : _type(oldType.remove_dimensions({ dimensionToRemove })), + _cellsResult(calcCellsSize(_type)), _innerDimSize(1), _sumDimSize(1), _outerDimSize(1) { - setup(dimensions, dimensionToRemove); + setup(oldType, dimensionToRemove); } template <typename Function> @@ -101,7 +86,7 @@ public: } assert(itr_out == _cellsResult.end()); assert(itr_in == cellsIn.cend()); - return std::make_unique<DenseTensor>(std::move(_dimensionsResult), std::move(_cellsResult)); + return std::make_unique<DenseTensor>(std::move(_type), std::move(_cellsResult)); } }; @@ -109,7 +94,7 @@ template <typename Function> DenseTensor::UP reduce(const DenseTensor &tensor, const vespalib::string &dimensionToRemove, Function &&func) { - DimensionReducer reducer(tensor.dimensionsMeta(), dimensionToRemove); + DimensionReducer reducer(tensor.type(), dimensionToRemove); return reducer.reduceCells(tensor.cells(), func); } diff --git a/vespalib/src/vespa/vespalib/tensor/dense/direct_dense_tensor_builder.cpp b/vespalib/src/vespa/vespalib/tensor/dense/direct_dense_tensor_builder.cpp index dd1682fb451..8a7ed1928ef 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/direct_dense_tensor_builder.cpp +++ b/vespalib/src/vespa/vespalib/tensor/dense/direct_dense_tensor_builder.cpp @@ -7,27 +7,27 @@ namespace vespalib { namespace tensor { using Address = DirectDenseTensorBuilder::Address; -using DimensionsMeta = DirectDenseTensorBuilder::DimensionsMeta; +using eval::ValueType; namespace { size_t -calculateCellsSize(const DimensionsMeta &dimensionsMeta) +calculateCellsSize(const ValueType &type) { size_t cellsSize = 1; - for (const auto &dimMeta : dimensionsMeta) { - cellsSize *= dimMeta.size(); + for (const auto &dim : type.dimensions()) { + cellsSize *= dim.size; } return cellsSize; } size_t -calculateCellAddress(const Address &address, const DimensionsMeta &dimensionsMeta) +calculateCellAddress(const Address &address, const ValueType &type) { - assert(address.size() == dimensionsMeta.size()); + assert(address.size() == type.dimensions().size()); size_t result = 0; for (size_t i = 0; i < address.size(); ++i) { - result *= dimensionsMeta[i].size(); + result *= type.dimensions()[i].size; result += address[i]; } return result; @@ -35,16 +35,16 @@ calculateCellAddress(const Address &address, const DimensionsMeta &dimensionsMet } -DirectDenseTensorBuilder::DirectDenseTensorBuilder(const DimensionsMeta &dimensionsMeta) - : _dimensionsMeta(dimensionsMeta), - _cells(calculateCellsSize(_dimensionsMeta)) +DirectDenseTensorBuilder::DirectDenseTensorBuilder(const ValueType &type_in) + : _type(type_in), + _cells(calculateCellsSize(_type)) { } void DirectDenseTensorBuilder::insertCell(const Address &address, double cellValue) { - size_t cellAddress = calculateCellAddress(address, _dimensionsMeta); + size_t cellAddress = calculateCellAddress(address, _type); assert(cellAddress < _cells.size()); _cells[cellAddress] = cellValue; } @@ -52,7 +52,7 @@ DirectDenseTensorBuilder::insertCell(const Address &address, double cellValue) Tensor::UP DirectDenseTensorBuilder::build() { - return std::make_unique<DenseTensor>(std::move(_dimensionsMeta), std::move(_cells)); + return std::make_unique<DenseTensor>(std::move(_type), std::move(_cells)); } } // namespace tensor diff --git a/vespalib/src/vespa/vespalib/tensor/dense/direct_dense_tensor_builder.h b/vespalib/src/vespa/vespalib/tensor/dense/direct_dense_tensor_builder.h index 74234f1cabe..b5329860e86 100644 --- a/vespalib/src/vespa/vespalib/tensor/dense/direct_dense_tensor_builder.h +++ b/vespalib/src/vespa/vespalib/tensor/dense/direct_dense_tensor_builder.h @@ -13,16 +13,15 @@ namespace tensor { class DirectDenseTensorBuilder { public: - using DimensionsMeta = DenseTensor::DimensionsMeta; using Cells = DenseTensor::Cells; using Address = std::vector<size_t>; private: - DimensionsMeta _dimensionsMeta; + eval::ValueType _type; Cells _cells; public: - DirectDenseTensorBuilder(const DimensionsMeta &dimensionsMeta); + DirectDenseTensorBuilder(const eval::ValueType &type_in); void insertCell(const Address &address, double cellValue); Tensor::UP build(); }; diff --git a/vespalib/src/vespa/vespalib/tensor/serialization/dense_binary_format.cpp b/vespalib/src/vespa/vespalib/tensor/serialization/dense_binary_format.cpp index f074f8d4335..0f6d1b3b2a5 100644 --- a/vespalib/src/vespa/vespalib/tensor/serialization/dense_binary_format.cpp +++ b/vespalib/src/vespa/vespalib/tensor/serialization/dense_binary_format.cpp @@ -11,16 +11,26 @@ using vespalib::nbostream; namespace vespalib { namespace tensor { +namespace { + +eval::ValueType +makeValueType(const std::vector<eval::ValueType::Dimension> &&dimensions) { + return (dimensions.empty() ? + eval::ValueType::double_type() : + eval::ValueType::tensor_type(std::move(dimensions))); +} + +} void DenseBinaryFormat::serialize(nbostream &stream, const DenseTensor &tensor) { - stream.putInt1_4Bytes(tensor.dimensionsMeta().size()); + stream.putInt1_4Bytes(tensor.type().dimensions().size()); size_t cellsSize = 1; - for (const auto &dimension : tensor.dimensionsMeta()) { - stream.writeSmallString(dimension.dimension()); - stream.putInt1_4Bytes(dimension.size()); - cellsSize *= dimension.size(); + for (const auto &dimension : tensor.type().dimensions()) { + stream.writeSmallString(dimension.name); + stream.putInt1_4Bytes(dimension.size); + cellsSize *= dimension.size; } const DenseTensor::Cells &cells = tensor.cells(); assert(cells.size() == cellsSize); @@ -34,15 +44,15 @@ std::unique_ptr<DenseTensor> DenseBinaryFormat::deserialize(nbostream &stream) { vespalib::string dimensionName; - DenseTensor::DimensionsMeta dimensionsMeta; + std::vector<eval::ValueType::Dimension> dimensions; DenseTensor::Cells cells; size_t dimensionsSize = stream.getInt1_4Bytes(); size_t dimensionSize; size_t cellsSize = 1; - while (dimensionsMeta.size() < dimensionsSize) { + while (dimensions.size() < dimensionsSize) { stream.readSmallString(dimensionName); dimensionSize = stream.getInt1_4Bytes(); - dimensionsMeta.emplace_back(dimensionName, dimensionSize); + dimensions.emplace_back(dimensionName, dimensionSize); cellsSize *= dimensionSize; } cells.reserve(cellsSize); @@ -51,7 +61,7 @@ DenseBinaryFormat::deserialize(nbostream &stream) stream >> cellValue; cells.emplace_back(cellValue); } - return std::make_unique<DenseTensor>(std::move(dimensionsMeta), + return std::make_unique<DenseTensor>(makeValueType(std::move(dimensions)), std::move(cells)); } diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/direct_sparse_tensor_builder.h b/vespalib/src/vespa/vespalib/tensor/sparse/direct_sparse_tensor_builder.h index 1d5b4b550a4..ad0257d8ec5 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/direct_sparse_tensor_builder.h +++ b/vespalib/src/vespa/vespalib/tensor/sparse/direct_sparse_tensor_builder.h @@ -18,14 +18,13 @@ template <> class DirectTensorBuilder<SparseTensor> { public: using TensorImplType = SparseTensor; - using Dimensions = typename TensorImplType::Dimensions; using Cells = typename TensorImplType::Cells; using AddressBuilderType = SparseTensorAddressBuilder; using AddressRefType = SparseTensorAddressRef; private: Stash _stash; - Dimensions _dimensions; + eval::ValueType _type; Cells _cells; public: @@ -40,10 +39,10 @@ public: } void - copyCells(const Cells &cells_in, const Dimensions &cells_in_dimensions) + copyCells(const Cells &cells_in, const eval::ValueType &cells_in_type) { - SparseTensorAddressPadder addressPadder(_dimensions, - cells_in_dimensions); + SparseTensorAddressPadder addressPadder(_type, + cells_in_type); for (const auto &cell : cells_in) { addressPadder.padAddress(cell.first); SparseTensorAddressRef oldRef = addressPadder.getAddressRef(); @@ -54,43 +53,43 @@ public: DirectTensorBuilder() : _stash(TensorImplType::STASH_CHUNK_SIZE), - _dimensions(), + _type(eval::ValueType::double_type()), _cells() { } - DirectTensorBuilder(const Dimensions &dimensions_in) + DirectTensorBuilder(const eval::ValueType &type_in) : _stash(TensorImplType::STASH_CHUNK_SIZE), - _dimensions(dimensions_in), + _type(type_in), _cells() { } - DirectTensorBuilder(const Dimensions &dimensions_in, + DirectTensorBuilder(const eval::ValueType &type_in, const Cells &cells_in) : _stash(TensorImplType::STASH_CHUNK_SIZE), - _dimensions(dimensions_in), + _type(type_in), _cells() { copyCells(cells_in); } - DirectTensorBuilder(const Dimensions &dimensions_in, + DirectTensorBuilder(const eval::ValueType &type_in, const Cells &cells_in, - const Dimensions &cells_dimensions) + const eval::ValueType &cells_in_type) : _stash(TensorImplType::STASH_CHUNK_SIZE), - _dimensions(dimensions_in), + _type(type_in), _cells() { - if (dimensions_in.size() == cells_dimensions.size()) { + if (type_in.dimensions().size() == cells_in_type.dimensions().size()) { copyCells(cells_in); } else { - copyCells(cells_in, cells_dimensions); + copyCells(cells_in, cells_in_type); } } Tensor::UP build() { - return std::make_unique<SparseTensor>(std::move(_dimensions), + return std::make_unique<SparseTensor>(std::move(_type), std::move(_cells), std::move(_stash)); } @@ -126,7 +125,7 @@ public: insertCell(address.getAddressRef(), value, [](double, double) -> double { abort(); }); } - Dimensions &dimensions() { return _dimensions; } + eval::ValueType &type() { return _type; } Cells &cells() { return _cells; } }; diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.cpp b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.cpp index 024d63572c6..ce7369e1aa0 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.cpp +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.cpp @@ -33,18 +33,18 @@ copyCells(Cells &cells, const Cells &cells_in, Stash &stash) void printAddress(std::ostream &out, const SparseTensorAddressRef &ref, - const TensorDimensions &dimensions) + const eval::ValueType &type) { out << "{"; bool first = true; SparseTensorAddressDecoder addr(ref); - for (auto &dim : dimensions) { + for (auto &dim : type.dimensions()) { auto label = addr.decodeLabel(); if (label.size() != 0u) { if (!first) { out << ","; } - out << dim << ":" << label; + out << dim.name << ":" << label; first = false; } } @@ -54,20 +54,20 @@ printAddress(std::ostream &out, const SparseTensorAddressRef &ref, } -SparseTensor::SparseTensor(const Dimensions &dimensions_in, - const Cells &cells_in) - : _cells(), - _dimensions(dimensions_in), +SparseTensor::SparseTensor(const eval::ValueType &type_in, + const Cells &cells_in) + : _type(type_in), + _cells(), _stash(STASH_CHUNK_SIZE) { copyCells(_cells, cells_in, _stash); } -SparseTensor::SparseTensor(Dimensions &&dimensions_in, - Cells &&cells_in, Stash &&stash_in) - : _cells(std::move(cells_in)), - _dimensions(std::move(dimensions_in)), +SparseTensor::SparseTensor(eval::ValueType &&type_in, + Cells &&cells_in, Stash &&stash_in) + : _type(std::move(type_in)), + _cells(std::move(cells_in)), _stash(std::move(stash_in)) { } @@ -76,29 +76,29 @@ SparseTensor::SparseTensor(Dimensions &&dimensions_in, bool SparseTensor::operator==(const SparseTensor &rhs) const { - return _dimensions == rhs._dimensions && _cells == rhs._cells; + return _type == rhs._type && _cells == rhs._cells; } -SparseTensor::Dimensions +eval::ValueType SparseTensor::combineDimensionsWith(const SparseTensor &rhs) const { - Dimensions result; - std::set_union(_dimensions.cbegin(), _dimensions.cend(), - rhs._dimensions.cbegin(), rhs._dimensions.cend(), - std::back_inserter(result)); - return result; + std::vector<eval::ValueType::Dimension> result; + std::set_union(_type.dimensions().cbegin(), _type.dimensions().cend(), + rhs._type.dimensions().cbegin(), rhs._type.dimensions().cend(), + std::back_inserter(result), + [](const eval::ValueType::Dimension &lhsDim, + const eval::ValueType::Dimension &rhsDim) + { return lhsDim.name < rhsDim.name; }); + return (result.empty() ? + eval::ValueType::double_type() : + eval::ValueType::tensor_type(result)); } eval::ValueType SparseTensor::getType() const { - if (_dimensions.empty()) { - return eval::ValueType::double_type(); - } - std::vector<eval::ValueType::Dimension> dimensions; - std::copy(_dimensions.begin(), _dimensions.end(), std::back_inserter(dimensions)); - return eval::ValueType::tensor_type(dimensions); + return _type; } double @@ -211,19 +211,19 @@ SparseTensor::toString() const Tensor::UP SparseTensor::clone() const { - return std::make_unique<SparseTensor>(_dimensions, _cells); + return std::make_unique<SparseTensor>(_type, _cells); } namespace { void -buildAddress(const SparseTensor::Dimensions &dimensions, +buildAddress(const eval::ValueType &type, SparseTensorAddressDecoder &decoder, TensorSpec::Address &address) { - for (const auto &dimension : dimensions) { + for (const auto &dimension : type.dimensions()) { auto label = decoder.decodeLabel(); - address.emplace(std::make_pair(dimension, TensorSpec::Label(label))); + address.emplace(std::make_pair(dimension.name, TensorSpec::Label(label))); } assert(!decoder.valid()); } @@ -237,11 +237,11 @@ SparseTensor::toSpec() const TensorSpec::Address address; for (const auto &cell : _cells) { SparseTensorAddressDecoder decoder(cell.first); - buildAddress(_dimensions, decoder, address); + buildAddress(_type, decoder, address); result.add(address, cell.second); address.clear(); } - if (_dimensions.empty() && _cells.empty()) { + if (_type.dimensions().empty() && _cells.empty()) { result.add(address, 0.0); } return result; @@ -256,7 +256,7 @@ SparseTensor::print(std::ostream &out) const if (!first) { out << ", "; } - printAddress(out, cell.first, _dimensions); + printAddress(out, cell.first, _type); out << ":" << cell.second; first = false; } @@ -271,10 +271,10 @@ SparseTensor::accept(TensorVisitor &visitor) const for (const auto &cell : _cells) { SparseTensorAddressDecoder decoder(cell.first); addrBuilder.clear(); - for (const auto &dimension : _dimensions) { + for (const auto &dimension : _type.dimensions()) { auto label = decoder.decodeLabel(); if (label.size() != 0u) { - addrBuilder.add(dimension, label); + addrBuilder.add(dimension.name, label); } } assert(!decoder.valid()); @@ -300,7 +300,7 @@ SparseTensor::reduce(const eval::BinaryOperation &op, const std::vector<vespalib::string> &dimensions) const { return sparse::reduce(*this, - (dimensions.empty() ? _dimensions : dimensions), + dimensions, [&op](double lhsValue, double rhsValue) { return op.eval(lhsValue, rhsValue); }); } diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.h b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.h index d788a55885e..5ed3d16b29c 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.h +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor.h @@ -23,24 +23,23 @@ class SparseTensor : public Tensor { public: typedef vespalib::hash_map<SparseTensorAddressRef, double> Cells; - typedef TensorDimensions Dimensions; static constexpr size_t STASH_CHUNK_SIZE = 16384u; private: + eval::ValueType _type; Cells _cells; - Dimensions _dimensions; Stash _stash; public: - explicit SparseTensor(const Dimensions &dimensions_in, + explicit SparseTensor(const eval::ValueType &type_in, const Cells &cells_in); - SparseTensor(Dimensions &&dimensions_in, + SparseTensor(eval::ValueType &&type_in, Cells &&cells_in, Stash &&stash_in); const Cells &cells() const { return _cells; } - const Dimensions &dimensions() const { return _dimensions; } + const eval::ValueType &type() const { return _type; } bool operator==(const SparseTensor &rhs) const; - Dimensions combineDimensionsWith(const SparseTensor &rhs) const; + eval::ValueType combineDimensionsWith(const SparseTensor &rhs) const; virtual eval::ValueType getType() const override; virtual double sum() const override; diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_combiner.cpp b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_combiner.cpp index 53cf90e2db0..1fa765aacfa 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_combiner.cpp +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_combiner.cpp @@ -3,22 +3,23 @@ #include <vespa/fastos/fastos.h> #include "sparse_tensor_address_combiner.h" #include "sparse_tensor_address_decoder.h" +#include <vespa/vespalib/eval/value_type.h> namespace vespalib { namespace tensor { namespace sparse { -TensorAddressCombiner::TensorAddressCombiner(const TensorDimensions &lhs, - const TensorDimensions &rhs) +TensorAddressCombiner::TensorAddressCombiner(const eval::ValueType &lhs, + const eval::ValueType &rhs) { - auto rhsItr = rhs.cbegin(); - auto rhsItrEnd = rhs.cend(); - for (auto &lhsDim : lhs) { - while (rhsItr != rhsItrEnd && *rhsItr < lhsDim) { + auto rhsItr = rhs.dimensions().cbegin(); + auto rhsItrEnd = rhs.dimensions().cend(); + for (auto &lhsDim : lhs.dimensions()) { + while (rhsItr != rhsItrEnd && rhsItr->name < lhsDim.name) { _ops.push_back(AddressOp::RHS); ++rhsItr; } - if (rhsItr != rhsItrEnd && *rhsItr == lhsDim) { + if (rhsItr != rhsItrEnd && rhsItr->name == lhsDim.name) { _ops.push_back(AddressOp::BOTH); ++rhsItr; } else { diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_combiner.h b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_combiner.h index 72717396a02..4340db30297 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_combiner.h +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_combiner.h @@ -6,6 +6,7 @@ #include <vespa/vespalib/tensor/types.h> namespace vespalib { +namespace eval { class ValueType; } namespace tensor { namespace sparse { @@ -25,8 +26,8 @@ class TensorAddressCombiner : public SparseTensorAddressBuilder std::vector<AddressOp> _ops; public: - TensorAddressCombiner(const TensorDimensions &lhs, - const TensorDimensions &rhs); + TensorAddressCombiner(const eval::ValueType &lhs, + const eval::ValueType &rhs); ~TensorAddressCombiner(); diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_padder.h b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_padder.h index 5f0c95033b3..abf73d5458e 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_padder.h +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_padder.h @@ -24,19 +24,21 @@ class SparseTensorAddressPadder : public SparseTensorAddressBuilder std::vector<PadOp> _padOps; public: - SparseTensorAddressPadder(const TensorDimensions &resultDims, - const TensorDimensions &inputDims) + SparseTensorAddressPadder(const eval::ValueType &resultType, + const eval::ValueType &inputType) : SparseTensorAddressBuilder(), _padOps() { - auto resultDimsItr = resultDims.cbegin(); - auto resultDimsItrEnd = resultDims.cend(); - for (auto &dim : inputDims) { - while (resultDimsItr != resultDimsItrEnd && *resultDimsItr < dim) { + auto resultDimsItr = resultType.dimensions().cbegin(); + auto resultDimsItrEnd = resultType.dimensions().cend(); + for (auto &dim : inputType.dimensions()) { + while (resultDimsItr != resultDimsItrEnd && + resultDimsItr->name < dim.name) { _padOps.push_back(PadOp::PAD); ++resultDimsItr; } - assert(resultDimsItr != resultDimsItrEnd && *resultDimsItr == dim); + assert(resultDimsItr != resultDimsItrEnd && + resultDimsItr->name == dim.name); _padOps.push_back(PadOp::COPY); ++resultDimsItr; } diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_reducer.cpp b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_reducer.cpp index 2d3bbaef043..6073acc4669 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_reducer.cpp +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_reducer.cpp @@ -2,12 +2,13 @@ #include <vespa/fastos/fastos.h> #include "sparse_tensor_address_reducer.h" +#include <vespa/vespalib/eval/value_type.h> namespace vespalib { namespace tensor { namespace sparse { -TensorAddressReducer::TensorAddressReducer(const TensorDimensions &dims, +TensorAddressReducer::TensorAddressReducer(const eval::ValueType &type, const std::vector<vespalib::string> & removeDimensions) : SparseTensorAddressBuilder(), @@ -15,9 +16,9 @@ TensorAddressReducer::TensorAddressReducer(const TensorDimensions &dims, { TensorDimensionsSet removeSet(removeDimensions.cbegin(), removeDimensions.cend()); - _ops.reserve(dims.size()); - for (auto &dim : dims) { - if (removeSet.find(dim) != removeSet.end()) { + _ops.reserve(type.dimensions().size()); + for (auto &dim : type.dimensions()) { + if (removeSet.find(dim.name) != removeSet.end()) { _ops.push_back(AddressOp::REMOVE); } else { _ops.push_back(AddressOp::COPY); @@ -25,23 +26,6 @@ TensorAddressReducer::TensorAddressReducer(const TensorDimensions &dims, } } -TensorDimensions -TensorAddressReducer::remainingDimensions(const TensorDimensions &dimensions, - const std::vector<vespalib::string> & - removeDimensions) -{ - TensorDimensionsSet removeSet(removeDimensions.cbegin(), - removeDimensions.cend()); - TensorDimensions result; - result.reserve(dimensions.size()); - for (auto &dim : dimensions) { - if (removeSet.find(dim) == removeSet.end()) { - result.push_back(dim); - } - } - return std::move(result); -} - TensorAddressReducer::~TensorAddressReducer() { } diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_reducer.h b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_reducer.h index 775607ca059..d92d83236c9 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_reducer.h +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_address_reducer.h @@ -7,6 +7,7 @@ #include "sparse_tensor_address_decoder.h" namespace vespalib { +namespace eval { class ValueType; } namespace tensor { namespace sparse { @@ -26,15 +27,11 @@ class TensorAddressReducer : public SparseTensorAddressBuilder AddressOps _ops; public: - TensorAddressReducer(const TensorDimensions &dims, + TensorAddressReducer(const eval::ValueType &type, const std::vector<vespalib::string> &removeDimensions); ~TensorAddressReducer(); - static TensorDimensions - remainingDimensions(const TensorDimensions &dimensions, - const std::vector<vespalib::string> &removeDimensions); - void reduce(SparseTensorAddressRef ref) { clear(); diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_apply.hpp b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_apply.hpp index 6c055d8547b..b32b09a01ac 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_apply.hpp +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_apply.hpp @@ -16,7 +16,7 @@ std::unique_ptr<Tensor> apply(const SparseTensor &lhs, const SparseTensor &rhs, Function &&func) { DirectTensorBuilder<SparseTensor> builder(lhs.combineDimensionsWith(rhs)); - TensorAddressCombiner addressCombiner(lhs.dimensions(), rhs.dimensions()); + TensorAddressCombiner addressCombiner(lhs.type(), rhs.type()); for (const auto &lhsCell : lhs.cells()) { for (const auto &rhsCell : rhs.cells()) { bool combineSuccess = addressCombiner.combine(lhsCell.first, diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_builder.cpp b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_builder.cpp index bb00d9b2e19..e9660665531 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_builder.cpp +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_builder.cpp @@ -14,7 +14,8 @@ SparseTensorBuilder::SparseTensorBuilder() _stash(SparseTensor::STASH_CHUNK_SIZE), _dimensionsEnum(), _dimensions(), - _sortedDimensions() + _type(eval::ValueType::double_type()), + _type_made(false) { } @@ -24,12 +25,19 @@ SparseTensorBuilder::~SparseTensorBuilder() void -SparseTensorBuilder::makeSortedDimensions() +SparseTensorBuilder::makeType() { - assert(_sortedDimensions.empty()); + assert(!_type_made); assert(_cells.empty()); - _sortedDimensions = _dimensions; - std::sort(_sortedDimensions.begin(), _sortedDimensions.end()); + std::vector<eval::ValueType::Dimension> dimensions; + dimensions.reserve(_dimensions.size()); + for (const auto &dim : _dimensions) { + dimensions.emplace_back(dim); + } + _type = (dimensions.empty() ? + eval::ValueType::double_type() : + eval::ValueType::tensor_type(dimensions)); + _type_made = true; } @@ -40,6 +48,7 @@ SparseTensorBuilder::define_dimension(const vespalib::string &dimension) if (it != _dimensionsEnum.end()) { return it->second; } + assert(!_type_made); Dimension res = _dimensionsEnum.size(); auto insres = _dimensionsEnum.insert(std::make_pair(dimension, res)); assert(insres.second); @@ -61,10 +70,10 @@ SparseTensorBuilder::add_label(Dimension dimension, TensorBuilder & SparseTensorBuilder::add_cell(double value) { - if (_dimensions.size() != _sortedDimensions.size()) { - makeSortedDimensions(); + if (!_type_made) { + makeType(); } - _addressBuilder.buildTo(_normalizedAddressBuilder, _sortedDimensions); + _addressBuilder.buildTo(_normalizedAddressBuilder, _type); SparseTensorAddressRef taddress(_normalizedAddressBuilder.getAddressRef()); // Make a persistent copy of sparse tensor address owned by _stash SparseTensorAddressRef address(taddress, _stash); @@ -79,18 +88,17 @@ Tensor::UP SparseTensorBuilder::build() { assert(_addressBuilder.empty()); - if (_dimensions.size() != _sortedDimensions.size()) { - makeSortedDimensions(); + if (!_type_made) { + makeType(); } - SparseTensor::Dimensions dimensions(_sortedDimensions.begin(), - _sortedDimensions.end()); - Tensor::UP ret = std::make_unique<SparseTensor>(std::move(dimensions), - std::move(_cells), - std::move(_stash)); + Tensor::UP ret = std::make_unique<SparseTensor>(std::move(_type), + std::move(_cells), + std::move(_stash)); SparseTensor::Cells().swap(_cells); _dimensionsEnum.clear(); _dimensions.clear(); - _sortedDimensions.clear(); + _type = eval::ValueType::double_type(); + _type_made = false; return ret; } diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_builder.h b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_builder.h index be0791a59c1..c6808614dd4 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_builder.h +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_builder.h @@ -24,9 +24,10 @@ class SparseTensorBuilder : public TensorBuilder Stash _stash; vespalib::hash_map<vespalib::string, uint32_t> _dimensionsEnum; std::vector<vespalib::string> _dimensions; - std::vector<vespalib::string> _sortedDimensions; + eval::ValueType _type; + bool _type_made; - void makeSortedDimensions(); + void makeType(); public: SparseTensorBuilder(); virtual ~SparseTensorBuilder(); diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_match.cpp b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_match.cpp index 35da291bbee..30cbad770a3 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_match.cpp +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_match.cpp @@ -19,17 +19,17 @@ enum class AddressOp void buildTransformOps(std::vector<AddressOp> &ops, - const TensorDimensions &lhs, - const TensorDimensions &rhs) + const eval::ValueType &lhs, + const eval::ValueType &rhs) { - auto rhsItr = rhs.cbegin(); - auto rhsItrEnd = rhs.cend(); - for (auto &lhsDim : lhs) { - while (rhsItr != rhsItrEnd && *rhsItr < lhsDim) { + auto rhsItr = rhs.dimensions().cbegin(); + auto rhsItrEnd = rhs.dimensions().cend(); + for (auto &lhsDim : lhs.dimensions()) { + while (rhsItr != rhsItrEnd && rhsItr->name < lhsDim.name) { ops.push_back(AddressOp::PAD); ++rhsItr; } - if (rhsItr != rhsItrEnd && *rhsItr == lhsDim) { + if (rhsItr != rhsItrEnd && rhsItr->name == lhsDim.name) { ops.push_back(AddressOp::COPY); ++rhsItr; } else { @@ -92,9 +92,9 @@ SparseTensorMatch::slowMatch(const TensorImplType &lhs, { std::vector<AddressOp> ops; SparseTensorAddressBuilder addressBuilder; - SparseTensorAddressPadder addressPadder(_builder.dimensions(), - lhs.dimensions()); - buildTransformOps(ops, lhs.dimensions(), rhs.dimensions()); + SparseTensorAddressPadder addressPadder(_builder.type(), + lhs.type()); + buildTransformOps(ops, lhs.type(), rhs.type()); for (const auto &lhsCell : lhs.cells()) { if (!transformAddress(addressBuilder, lhsCell.first, ops)) { continue; @@ -112,8 +112,8 @@ SparseTensorMatch::SparseTensorMatch(const TensorImplType &lhs, const TensorImplType &rhs) : Parent(lhs.combineDimensionsWith(rhs)) { - if ((lhs.dimensions().size() == rhs.dimensions().size()) && - (lhs.dimensions().size() == _builder.dimensions().size())) { + if ((lhs.type().dimensions().size() == rhs.type().dimensions().size()) && + (lhs.type().dimensions().size() == _builder.type().dimensions().size())) { fastMatch(lhs, rhs); } else { slowMatch(lhs, rhs); diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_reduce.hpp b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_reduce.hpp index 45e6b727881..a92b9caa08c 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_reduce.hpp +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_reduce.hpp @@ -45,11 +45,11 @@ reduce(const SparseTensor &tensor, if (dimensions.empty()) { return reduceAll(tensor, func); } - DirectTensorBuilder<SparseTensor> builder(TensorAddressReducer::remainingDimensions(tensor.dimensions(), dimensions)); - if (builder.dimensions().empty()) { + DirectTensorBuilder<SparseTensor> builder(tensor.type().remove_dimensions(dimensions)); + if (builder.type().dimensions().empty()) { return reduceAll(tensor, builder, func); } - TensorAddressReducer addressReducer(tensor.dimensions(), dimensions); + TensorAddressReducer addressReducer(tensor.type(), dimensions); for (const auto &cell : tensor.cells()) { addressReducer.reduce(cell.first); builder.insertCell(addressReducer.getAddressRef(), cell.second, func); diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp index 57db0902396..9361cbcf7f8 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_unsorted_address_builder.cpp @@ -3,6 +3,7 @@ #include <vespa/fastos/fastos.h> #include "sparse_tensor_unsorted_address_builder.h" #include "sparse_tensor_address_builder.h" +#include <vespa/vespalib/eval/value_type.h> #include <algorithm> namespace vespalib { @@ -17,25 +18,24 @@ SparseTensorUnsortedAddressBuilder::SparseTensorUnsortedAddressBuilder() void SparseTensorUnsortedAddressBuilder::buildTo(SparseTensorAddressBuilder & - builder, - const TensorDimensions & - dimensions) + builder, + const eval::ValueType &type) { const char *base = &_elementStrings[0]; std::sort(_elements.begin(), _elements.end(), [=](const ElementRef &lhs, const ElementRef &rhs) { return lhs.getDimension(base) < rhs.getDimension(base); }); // build normalized address with sorted dimensions - auto dimsItr = dimensions.cbegin(); - auto dimsItrEnd = dimensions.cend(); + auto dimsItr = type.dimensions().cbegin(); + auto dimsItrEnd = type.dimensions().cend(); for (const auto &element : _elements) { while ((dimsItr != dimsItrEnd) && - (*dimsItr < element.getDimension(base))) { + (dimsItr->name < element.getDimension(base))) { builder.addUndefined(); ++dimsItr; } assert((dimsItr != dimsItrEnd) && - (*dimsItr == element.getDimension(base))); + (dimsItr->name == element.getDimension(base))); builder.add(element.getLabel(base)); ++dimsItr; } diff --git a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_unsorted_address_builder.h b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_unsorted_address_builder.h index 914f7d6ce2f..5fcf9590a89 100644 --- a/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_unsorted_address_builder.h +++ b/vespalib/src/vespa/vespalib/tensor/sparse/sparse_tensor_unsorted_address_builder.h @@ -7,6 +7,7 @@ #include <vespa/vespalib/tensor/types.h> namespace vespalib { +namespace eval { class ValueType; } namespace tensor { class SparseTensorAddressBuilder; @@ -72,7 +73,7 @@ public: * tensor address builder in sorted order. */ void buildTo(SparseTensorAddressBuilder &builder, - const TensorDimensions &dimensions); + const eval::ValueType &type); void clear() { _elementStrings.clear(); _elements.clear(); } }; diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_apply.cpp b/vespalib/src/vespa/vespalib/tensor/tensor_apply.cpp index 7f0293f6349..23edf418c0b 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_apply.cpp +++ b/vespalib/src/vespa/vespalib/tensor/tensor_apply.cpp @@ -9,7 +9,7 @@ namespace tensor { template <class TensorT> TensorApply<TensorT>::TensorApply(const TensorImplType &tensor, const CellFunction &func) - : Parent(tensor.dimensions()) + : Parent(tensor.type()) { for (const auto &cell : tensor.cells()) { _builder.insertCell(cell.first, func.apply(cell.second)); diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp index a527627d786..f740ffbf348 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp +++ b/vespalib/src/vespa/vespalib/tensor/tensor_mapper.cpp @@ -17,25 +17,8 @@ namespace tensor { namespace { -class SparseTensorMapperBase -{ -protected: - static TensorDimensions mapDimensions(const ValueType &type); -}; - -TensorDimensions -SparseTensorMapperBase::mapDimensions(const ValueType &type) -{ - TensorDimensions dimensions; - dimensions.reserve(type.dimensions().size()); - for (const auto &dimension : type.dimensions()) { - dimensions.emplace_back(dimension.name); - } - return dimensions; -} - template <class TensorT> -class SparseTensorMapper : public TensorVisitor, public SparseTensorMapperBase +class SparseTensorMapper : public TensorVisitor { using Builder = DirectTensorBuilder<TensorT>; using AddressBuilderType = typename Builder::AddressBuilderType; @@ -60,8 +43,7 @@ template <class TensorT> SparseTensorMapper<TensorT>:: SparseTensorMapper(const ValueType &type) : TensorVisitor(), - SparseTensorMapperBase(), - _builder(mapDimensions(type)), + _builder(type), _addressBuilder() { } @@ -85,8 +67,8 @@ mapAddress(const TensorAddress &address) { _addressBuilder.clear(); TensorAddressElementIterator<TensorAddress> addressIterator(address); - for (const auto &dimension : _builder.dimensions()) { - if (addressIterator.skipToDimension(dimension)) { + for (const auto &dimension : _builder.type().dimensions()) { + if (addressIterator.skipToDimension(dimension.name)) { _addressBuilder.add(addressIterator.label()); addressIterator.next(); } else { @@ -118,7 +100,7 @@ SparseTensorMapper<TensorT>::map(const Tensor &tensor, class DenseTensorMapper : public TensorVisitor { - DenseTensor::DimensionsMeta _dimensionsMeta; + eval::ValueType _type; DenseTensor::Cells _cells; static constexpr uint32_t BAD_LABEL = std::numeric_limits<uint32_t>::max(); static constexpr uint32_t BAD_ADDRESS = @@ -138,14 +120,12 @@ public: }; DenseTensorMapper::DenseTensorMapper(const ValueType &type) - : _dimensionsMeta(), + : _type(type), _cells() { - _dimensionsMeta.reserve(type.dimensions().size()); size_t size = 1; for (const auto &dimension : type.dimensions()) { size *= dimension.size; - _dimensionsMeta.emplace_back(dimension.name, dimension.size); } _cells.resize(size); } @@ -157,7 +137,7 @@ DenseTensorMapper::~DenseTensorMapper() std::unique_ptr<Tensor> DenseTensorMapper::build() { - return std::make_unique<DenseTensor>(std::move(_dimensionsMeta), + return std::make_unique<DenseTensor>(std::move(_type), std::move(_cells)); } @@ -182,17 +162,17 @@ DenseTensorMapper::mapAddressToIndex(const TensorAddress &address) { uint32_t idx = 0; TensorAddressElementIterator<TensorAddress> addressIterator(address); - for (const auto &dimension : _dimensionsMeta) { - if (addressIterator.skipToDimension(dimension.dimension())) { + for (const auto &dimension : _type.dimensions()) { + if (addressIterator.skipToDimension(dimension.name)) { uint32_t label = mapLabelToNumber(addressIterator.label()); - if (label == BAD_LABEL || label >= dimension.size()) { + if (label == BAD_LABEL || label >= dimension.size) { return BAD_ADDRESS; } - idx = idx * dimension.size() + label; + idx = idx * dimension.size + label; addressIterator.next(); } else { // output dimension not in input - idx = idx * dimension.size(); + idx = idx * dimension.size; } } return idx; diff --git a/vespalib/src/vespa/vespalib/tensor/tensor_operation.h b/vespalib/src/vespa/vespalib/tensor/tensor_operation.h index 350dfcc8abc..c4fc88f3b5e 100644 --- a/vespalib/src/vespa/vespalib/tensor/tensor_operation.h +++ b/vespalib/src/vespa/vespalib/tensor/tensor_operation.h @@ -17,29 +17,28 @@ class TensorOperation public: using TensorImplType = TensorT; using MyTensorBuilder = DirectTensorBuilder<TensorT>; - using Dimensions = typename TensorImplType::Dimensions; using Cells = typename TensorImplType::Cells; using AddressBuilderType = typename MyTensorBuilder::AddressBuilderType; using AddressRefType = typename MyTensorBuilder::AddressRefType; protected: MyTensorBuilder _builder; - Dimensions &_dimensions; + eval::ValueType &_type; Cells &_cells; public: TensorOperation() : _builder(), - _dimensions(_builder.dimensions()), + _type(_builder.type()), _cells(_builder.cells()) {} - TensorOperation(const Dimensions &dimensions) - : _builder(dimensions), - _dimensions(_builder.dimensions()), + TensorOperation(const eval::ValueType &type) + : _builder(type), + _type(_builder.type()), _cells(_builder.cells()) {} - TensorOperation(const Dimensions &dimensions, const Cells &cells) - : _builder(dimensions, cells), - _dimensions(_builder.dimensions()), + TensorOperation(const eval::ValueType &type, const Cells &cells) + : _builder(type, cells), + _type(_builder.type()), _cells(_builder.cells()) {} Tensor::UP result() { diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp index b037535a635..e95a9d2d652 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.cpp +++ b/vespalib/src/vespa/vespalib/util/alloc.cpp @@ -116,17 +116,17 @@ namespace alloc { class HeapAllocator : public MemoryAllocator { public: - void * alloc(size_t sz) const override; - void free(void * buf, size_t sz) const override; - static void * salloc(size_t sz); - static void sfree(void * buf, size_t sz); + PtrAndSize alloc(size_t sz) const override; + void free(PtrAndSize alloc) const override; + static PtrAndSize salloc(size_t sz); + static void sfree(PtrAndSize alloc); static MemoryAllocator & getDefault(); }; class AlignedHeapAllocator : public HeapAllocator { public: AlignedHeapAllocator(size_t alignment) : _alignment(alignment) { } - void * alloc(size_t sz) const override; + PtrAndSize alloc(size_t sz) const override; static MemoryAllocator & get4K(); static MemoryAllocator & get1K(); static MemoryAllocator & get512B(); @@ -136,18 +136,18 @@ private: class MMapAllocator : public MemoryAllocator { public: - void * alloc(size_t sz) const override; - void free(void * buf, size_t sz) const override; - static void * salloc(size_t sz); - static void sfree(void * buf, size_t sz); + PtrAndSize alloc(size_t sz) const override; + void free(PtrAndSize alloc) const override; + static PtrAndSize salloc(size_t sz); + static void sfree(PtrAndSize alloc); static MemoryAllocator & getDefault(); }; class AutoAllocator : public MemoryAllocator { public: AutoAllocator(size_t mmapLimit, size_t alignment) : _mmapLimit(mmapLimit), _alignment(alignment) { } - void * alloc(size_t sz) const override; - void free(void * buf, size_t sz) const override; + PtrAndSize alloc(size_t sz) const override; + void free(PtrAndSize alloc) const override; static MemoryAllocator & getDefault(); static MemoryAllocator & getAllocator(size_t mmapLimit, size_t alignment); private: @@ -226,37 +226,42 @@ MemoryAllocator & AutoAllocator::getAllocator(size_t mmapLimit, size_t alignment return *(found->second); } -void * HeapAllocator::alloc(size_t sz) const { +MemoryAllocator::PtrAndSize +HeapAllocator::alloc(size_t sz) const { return salloc(sz); } -void * HeapAllocator::salloc(size_t sz) { - return (sz > 0) ? malloc(sz) : 0; +MemoryAllocator::PtrAndSize +HeapAllocator::salloc(size_t sz) { + return PtrAndSize((sz > 0) ? malloc(sz) : nullptr, sz); } -void HeapAllocator::free(void * p, size_t sz) const { - sfree(p, sz); +void HeapAllocator::free(PtrAndSize alloc) const { + sfree(alloc); } -void HeapAllocator::sfree(void * p, size_t sz) { - (void) sz; if (p) { ::free(p); } +void HeapAllocator::sfree(PtrAndSize alloc) { + if (alloc.first) { ::free(alloc.first); } } -void * AlignedHeapAllocator::alloc(size_t sz) const { - if (!sz) { return 0; } +MemoryAllocator::PtrAndSize +AlignedHeapAllocator::alloc(size_t sz) const { + if (!sz) { return PtrAndSize(nullptr, 0); } void* ptr; int result = posix_memalign(&ptr, _alignment, sz); if (result != 0) { throw IllegalArgumentException(make_string("posix_memalign(%zu, %zu) failed with code %d", sz, _alignment, result)); } - return ptr; + return PtrAndSize(ptr, sz); } -void * MMapAllocator::alloc(size_t sz) const { +MemoryAllocator::PtrAndSize +MMapAllocator::alloc(size_t sz) const { return salloc(sz); } -void * MMapAllocator::salloc(size_t sz) +MemoryAllocator::PtrAndSize +MMapAllocator::salloc(size_t sz) { void * buf(nullptr); if (sz > 0) { @@ -304,29 +309,31 @@ void * MMapAllocator::salloc(size_t sz) LOG(info, "%ld mappings of accumulated size %ld", _G_HugeMappings.size(), sum(_G_HugeMappings)); } } - return buf; + return PtrAndSize(buf, sz); } -void MMapAllocator::free(void * buf, size_t sz) const { - sfree(buf, sz); +void MMapAllocator::free(PtrAndSize alloc) const { + sfree(alloc); } -void MMapAllocator::sfree(void * buf, size_t sz) +void MMapAllocator::sfree(PtrAndSize alloc) { - if (buf != nullptr) { - madvise(buf, sz, MADV_DONTNEED); - munmap(buf, sz); - if (sz >= _G_MMapLogLimit) { + if (alloc.first != nullptr) { + madvise(alloc.first, alloc.second, MADV_DONTNEED); + munmap(alloc.first, alloc.second); + if (alloc.second >= _G_MMapLogLimit) { LockGuard guard(_G_lock); - MMapInfo info = _G_HugeMappings[buf]; - _G_HugeMappings.erase(buf); + MMapInfo info = _G_HugeMappings[alloc.first]; + assert(alloc.second == info._sz); + _G_HugeMappings.erase(alloc.first); LOG(info, "munmap %ld of size %ld", info._id, info._sz); LOG(info, "%ld mappings of accumulated size %ld", _G_HugeMappings.size(), sum(_G_HugeMappings)); } } } -void * AutoAllocator::alloc(size_t sz) const { +MMapAllocator::PtrAndSize +AutoAllocator::alloc(size_t sz) const { if (useMMap(sz)) { sz = roundUpToHugePages(sz); return MMapAllocator::salloc(sz); @@ -339,22 +346,23 @@ void * AutoAllocator::alloc(size_t sz) const { } } -void AutoAllocator::free(void *p, size_t sz) const { - if (useMMap(sz)) { - return MMapAllocator::sfree(p, sz); +void +AutoAllocator::free(PtrAndSize alloc) const { + if (useMMap(alloc.second)) { + return MMapAllocator::sfree(alloc); } else { - return HeapAllocator::sfree(p, sz); + return HeapAllocator::sfree(alloc); } } Alloc -HeapAllocFactory::create(size_t sz) +Alloc::allocHeap(size_t sz) { return Alloc(&HeapAllocator::getDefault(), sz); } Alloc -AlignedHeapAllocFactory::create(size_t sz, size_t alignment) +Alloc::allocAlignedHeap(size_t sz, size_t alignment) { if (alignment == 0) { return Alloc(&AlignedHeapAllocator::getDefault(), sz); @@ -365,18 +373,18 @@ AlignedHeapAllocFactory::create(size_t sz, size_t alignment) } else if (alignment == 0x1000) { return Alloc(&AlignedHeapAllocator::get4K(), sz); } else { - throw IllegalArgumentException(make_string("AlignedHeapAllocFactory::create(%zu, %zu) does not support %zu alignment", sz, alignment, alignment)); + throw IllegalArgumentException(make_string("Alloc::allocAlignedHeap(%zu, %zu) does not support %zu alignment", sz, alignment, alignment)); } } Alloc -MMapAllocFactory::create(size_t sz) +Alloc::allocMMap(size_t sz) { return Alloc(&MMapAllocator::getDefault(), sz); } Alloc -AutoAllocFactory::create(size_t sz, size_t mmapLimit, size_t alignment) +Alloc::alloc(size_t sz, size_t mmapLimit, size_t alignment) { return Alloc(&AutoAllocator::getAllocator(mmapLimit, alignment), sz); } diff --git a/vespalib/src/vespa/vespalib/util/alloc.h b/vespalib/src/vespa/vespalib/util/alloc.h index 74366439f2c..b8b2374c109 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.h +++ b/vespalib/src/vespa/vespalib/util/alloc.h @@ -3,7 +3,6 @@ #include <sys/types.h> #include <algorithm> -#include <vespa/vespalib/util/linkedptr.h> #include <vespa/vespalib/util/optimized.h> namespace vespalib { @@ -14,12 +13,13 @@ class MemoryAllocator { public: enum {HUGEPAGE_SIZE=0x200000}; using UP = std::unique_ptr<MemoryAllocator>; + using PtrAndSize = std::pair<void *, size_t>; MemoryAllocator(const MemoryAllocator &) = delete; MemoryAllocator & operator = (const MemoryAllocator &) = delete; MemoryAllocator() { } virtual ~MemoryAllocator() { } - virtual void * alloc(size_t sz) const = 0; - virtual void free(void * buf, size_t sz) const = 0; + virtual PtrAndSize alloc(size_t sz) const = 0; + virtual void free(PtrAndSize alloc) const = 0; static size_t roundUpToHugePages(size_t sz) { return (sz+(HUGEPAGE_SIZE-1)) & ~(HUGEPAGE_SIZE-1); } @@ -33,80 +33,65 @@ public: **/ class Alloc { +private: + using PtrAndSize = MemoryAllocator::PtrAndSize;; public: - using MemoryAllocator = alloc::MemoryAllocator; - size_t size() const { return _sz; } - void * get() { return _buf; } - const void * get() const { return _buf; } - void * operator -> () { return _buf; } - const void * operator -> () const { return _buf; } + size_t size() const { return _alloc.second; } + void * get() { return _alloc.first; } + const void * get() const { return _alloc.first; } + void * operator -> () { return _alloc.first; } + const void * operator -> () const { return _alloc.first; } Alloc(const Alloc &) = delete; Alloc & operator = (const Alloc &) = delete; Alloc(Alloc && rhs) : - _buf(rhs._buf), - _sz(rhs._sz), + _alloc(rhs._alloc), _allocator(rhs._allocator) { - rhs._buf = nullptr; - rhs._sz = 0; - rhs._allocator = 0; + rhs.clear(); } Alloc & operator=(Alloc && rhs) { if (this != & rhs) { - swap(rhs); + if (_alloc.first != nullptr) { + _allocator->free(_alloc); + } + _alloc = rhs._alloc; + _allocator = rhs._allocator; + rhs.clear(); } return *this; } - Alloc() : _buf(nullptr), _sz(0), _allocator(nullptr) { } - Alloc(const MemoryAllocator * allocator, size_t sz) : _buf(allocator->alloc(sz)), _sz(sz), _allocator(allocator) { } + Alloc() : _alloc(nullptr, 0), _allocator(nullptr) { } ~Alloc() { - if (_buf != nullptr) { - _allocator->free(_buf, _sz); - _buf = nullptr; + if (_alloc.first != nullptr) { + _allocator->free(_alloc); + _alloc.first = nullptr; } } void swap(Alloc & rhs) { - std::swap(_buf, rhs._buf); - std::swap(_sz, rhs._sz); + std::swap(_alloc, rhs._alloc); std::swap(_allocator, rhs._allocator); } Alloc create(size_t sz) const { return Alloc(_allocator, sz); } -protected: - void * _buf; - size_t _sz; - const MemoryAllocator * _allocator; -}; - -class HeapAllocFactory -{ -public: - static Alloc create(size_t sz=0); -}; - -class AlignedHeapAllocFactory -{ -public: - static Alloc create(size_t sz, size_t alignment); -}; - -class MMapAllocFactory -{ -public: - enum {HUGEPAGE_SIZE=0x200000}; - static Alloc create(size_t sz=0); -}; - -/** - * Optional alignment is assumed to be <= system page size, since mmap - * is always used when size is above limit. - */ -class AutoAllocFactory -{ -public: - static Alloc create(size_t sz=0, size_t mmapLimit=MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0); + static Alloc allocAlignedHeap(size_t sz, size_t alignment); + static Alloc allocHeap(size_t sz=0); + static Alloc allocMMap(size_t sz=0); + /** + * Optional alignment is assumed to be <= system page size, since mmap + * is always used when size is above limit. + */ + static Alloc alloc(size_t sz=0, size_t mmapLimit=MemoryAllocator::HUGEPAGE_SIZE, size_t alignment=0); +private: + Alloc(const MemoryAllocator * allocator, size_t sz) : _alloc(allocator->alloc(sz)), _allocator(allocator) { } + void clear() { + _alloc.first = nullptr; + _alloc.second = 0; + _allocator = nullptr; + } + PtrAndSize _alloc; + const MemoryAllocator * _allocator; }; } @@ -115,6 +100,4 @@ inline size_t roundUp2inN(size_t minimum) { return 2ul << Optimized::msbIdx(minimum - 1); } -using DefaultAlloc = alloc::AutoAllocFactory; - } diff --git a/vespalib/src/vespa/vespalib/util/array.h b/vespalib/src/vespa/vespalib/util/array.h index f2546e46e22..4df60d68cdc 100644 --- a/vespalib/src/vespa/vespalib/util/array.h +++ b/vespalib/src/vespa/vespalib/util/array.h @@ -108,12 +108,12 @@ public: typedef T value_type; typedef size_t size_type; - Array(const Alloc & initial=DefaultAlloc::create()) : _array(initial.create(0)), _sz(0) { } - Array(size_t sz, const Alloc & initial=DefaultAlloc::create()); + Array(const Alloc & initial=Alloc::alloc()) : _array(initial.create(0)), _sz(0) { } + Array(size_t sz, const Alloc & initial=Alloc::alloc()); Array(Alloc && buf, size_t sz); Array(Array &&rhs); - Array(size_t sz, T value, const Alloc & initial=DefaultAlloc::create()); - Array(const_iterator begin, const_iterator end, const Alloc & initial=DefaultAlloc::create()); + Array(size_t sz, T value, const Alloc & initial=Alloc::alloc()); + Array(const_iterator begin, const_iterator end, const Alloc & initial=Alloc::alloc()); Array(const Array & rhs); Array & operator =(const Array & rhs) { if (&rhs != this) { |