diff options
94 files changed, 807 insertions, 1169 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelProvider.java b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelProvider.java index 36702f6e97c..5b636385b6a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelProvider.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/SuperModelProvider.java @@ -13,6 +13,6 @@ public interface SuperModelProvider { */ SuperModel snapshot(SuperModelListener listener); - // TODO: Remove when 6.306 is latest version in use + // TODO: Remove when 6.313 is latest version in use Zone getZone(); } diff --git a/config-model/pom.xml b/config-model/pom.xml index e8415b3a67b..a65a6a836ed 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -299,12 +299,10 @@ <dependency> <groupId>org.tensorflow</groupId> <artifactId>proto</artifactId> - <version>1.6.0</version> </dependency> <dependency> <groupId>org.tensorflow</groupId> <artifactId>tensorflow</artifactId> - <version>1.6.0</version> </dependency> </dependencies> diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java index d7c4c27b2b0..57ee5da5c54 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Search.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/Search.java @@ -582,19 +582,13 @@ public class Search implements Serializable, ImmutableSearch { return false; } - /** - * The field set settings for this search - * - * @return field set settings for this - */ - public FieldSets fieldSets() { - return fieldSets; - } + /** The field set settings for this search */ + public FieldSets fieldSets() { return fieldSets; } /** * For adding structs defined in document scope * - * @param dt The struct to add. + * @param dt the struct to add * @return self, for chaining */ public Search addType(SDDocumentType dt) { @@ -606,4 +600,5 @@ public class Search implements Serializable, ImmutableSearch { docType.addAnnotation(dt); return this; } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java index 2e9c2a6e3f3..67b3447220a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java @@ -295,7 +295,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { builder.indexinfo(iiB); } - // TODO: Move this to the FieldSetValidity processor (and rename it) as that already has to look at this. + // TODO: Move this to the FieldSetSettings processor (and rename it) as that already has to look at this. private void addFieldSetCommands(IndexInfoConfig.Indexinfo.Builder iiB, FieldSet fieldSet) { // Explicit query commands on the field set, overrides everything. if (!fieldSet.queryCommands().isEmpty()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetSettings.java index 4ef9a9733d5..21647bdfcf7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetValidity.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetSettings.java @@ -17,43 +17,46 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; * @author vegardh * @author bratseth */ -public class FieldSetValidity extends Processor { +// See also IndexInfo.addFieldSetCommands, which does more of this in a complicated way. +// That should be moved here, and done in the way the match setting is done below +// (this requires adding normalizing and stemming settings to FieldSet). +public class FieldSetSettings extends Processor { - public FieldSetValidity(Search search, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, QueryProfiles queryProfiles) { + public FieldSetSettings(Search search, + DeployLogger deployLogger, + RankProfileRegistry rankProfileRegistry, + QueryProfiles queryProfiles) { super(search, deployLogger, rankProfileRegistry, queryProfiles); } @Override public void process(boolean validate, boolean documentsOnly) { - if ( ! validate) return; - for (FieldSet fieldSet : search.fieldSets().userFieldSets().values()) { - checkFieldNames(search, fieldSet); + if (validate) + checkFieldNames(search, fieldSet); checkMatching(search, fieldSet); checkNormalization(search, fieldSet); checkStemming(search, fieldSet); } } - private static void checkFieldNames(Search search, FieldSet fieldSet) { - for (String fld : fieldSet.getFieldNames()) { - ImmutableSDField field = search.getField(fld); - if (field == null) { - throw new IllegalArgumentException( - "For search '" + search.getName() + "': Field '"+ fld + "' in " + fieldSet + " does not exist."); - } + private void checkFieldNames(Search search, FieldSet fieldSet) { + for (String field : fieldSet.getFieldNames()) { + if (search.getField(field) == null) + throw new IllegalArgumentException("For search '" + search.getName() + + "': Field '"+ field + "' in " + fieldSet + " does not exist."); } } private void checkMatching(Search search, FieldSet fieldSet) { - Matching fsMatching = null; - for (String fld : fieldSet.getFieldNames()) { - ImmutableSDField field = search.getField(fld); + Matching matching = fieldSet.getMatching(); + for (String fieldName : fieldSet.getFieldNames()) { + ImmutableSDField field = search.getField(fieldName); Matching fieldMatching = field.getMatching(); - if (fsMatching==null) { - fsMatching = fieldMatching; + if (matching == null) { + matching = fieldMatching; } else { - if ( ! fsMatching.equals(fieldMatching)) { + if ( ! matching.equals(fieldMatching)) { warn(search, field.asField(), "The matching settings for the fields in " + fieldSet + " are inconsistent " + "(explicitly or because of field type). This may lead to recall and ranking issues."); @@ -61,17 +64,18 @@ public class FieldSetValidity extends Processor { } } } + fieldSet.setMatching(matching); // Assign the uniquely determined matching to the field set } private void checkNormalization(Search search, FieldSet fieldSet) { - NormalizeLevel.Level fsNorm = null; - for (String fld : fieldSet.getFieldNames()) { - ImmutableSDField field = search.getField(fld); + NormalizeLevel.Level normalizing = null; + for (String fieldName : fieldSet.getFieldNames()) { + ImmutableSDField field = search.getField(fieldName); NormalizeLevel.Level fieldNorm = field.getNormalizing().getLevel(); - if (fsNorm==null) { - fsNorm = fieldNorm; + if (normalizing == null) { + normalizing = fieldNorm; } else { - if ( ! fsNorm.equals(fieldNorm)) { + if ( ! normalizing.equals(fieldNorm)) { warn(search, field.asField(), "The normalization settings for the fields in " + fieldSet + " are inconsistent " + "(explicitly or because of field type). This may lead to recall and ranking issues."); @@ -82,14 +86,14 @@ public class FieldSetValidity extends Processor { } private void checkStemming(Search search, FieldSet fieldSet) { - Stemming fsStemming = null; - for (String fld : fieldSet.getFieldNames()) { - ImmutableSDField field = search.getField(fld); + Stemming stemming = null; + for (String fieldName : fieldSet.getFieldNames()) { + ImmutableSDField field = search.getField(fieldName); Stemming fieldStemming = field.getStemming(); - if (fsStemming==null) { - fsStemming = fieldStemming; + if (stemming == null) { + stemming = fieldStemming; } else { - if ( ! fsStemming.equals(fieldStemming)) { + if ( ! stemming.equals(fieldStemming)) { warn(search, field.asField(), "The stemming settings for the fields in the fieldset '"+fieldSet.getName()+ "' are inconsistent (explicitly or because of field type). " + diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java index f744eeb82e1..062b82bd230 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java @@ -65,7 +65,7 @@ public class Processing { SummaryDiskAccessValidator::new, DisallowComplexMapAndWsetKeyTypes::new, SortingSettings::new, - FieldSetValidity::new, + FieldSetSettings::new, AddExtraFieldsToDocument::new, PredicateProcessor::new, MatchPhaseSettingsValidator::new, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java index 3db09f6b566..a51e9e40c61 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/SystemMetrics.java @@ -18,6 +18,8 @@ public class SystemMetrics { public static final String MEM_LIMIT = "mem.limit"; public static final String MEM_USED = "mem.used"; public static final String MEM_UTIL = "mem.util"; + public static final String MEM_TOTAL_USED = "mem_total.used"; + public static final String MEM_TOTAL_UTIL = "mem_total.util"; public static final MetricSet systemMetricSet = createSystemMetricSet(); @@ -30,7 +32,9 @@ public class SystemMetrics { new Metric(DISK_UTIL), new Metric(MEM_LIMIT), new Metric(MEM_USED), - new Metric(MEM_UTIL) + new Metric(MEM_UTIL), + new Metric(MEM_TOTAL_USED), + new Metric(MEM_TOTAL_UTIL) ); Set<Metric> nonDockerNodeMetrics = diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java index f6f64eba482..b845b180548 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java @@ -5,7 +5,6 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.vespa.config.search.core.ProtonConfig; import static java.lang.Long.min; -import static java.lang.Integer.max; /** * Tuning of proton config for a search node based on the node flavor of that node. @@ -31,6 +30,7 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { tuneFlushStrategyTlsSize(builder.flush.memory); tuneSummaryReadIo(builder.summary.read); tuneSummaryCache(builder.summary.cache); + tuneSearchReadIo(builder.search.mmap); } private void tuneSummaryCache(ProtonConfig.Summary.Cache.Builder builder) { @@ -82,4 +82,10 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { } } + private void tuneSearchReadIo(ProtonConfig.Search.Mmap.Builder builder) { + if (nodeFlavor.hasFastDisk()) { + builder.advise(ProtonConfig.Search.Mmap.Advise.RANDOM); + } + } + } diff --git a/config-model/src/test/derived/fieldset/index-info.cfg b/config-model/src/test/derived/fieldset/index-info.cfg new file mode 100644 index 00000000000..47dcbe58805 --- /dev/null +++ b/config-model/src/test/derived/fieldset/index-info.cfg @@ -0,0 +1,33 @@ +indexinfo[].name "test" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "sddocname" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "word1" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "word1" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "word1" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "word1" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "word2" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "word2" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "word2" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "word2" +indexinfo[].command[].command "word" +indexinfo[].command[].indexname "rankfeatures" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "summaryfeatures" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "words" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "words" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "words" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "words" +indexinfo[].command[].command "word" diff --git a/config-model/src/test/derived/fieldset/test.sd b/config-model/src/test/derived/fieldset/test.sd new file mode 100644 index 00000000000..c5ed169cc94 --- /dev/null +++ b/config-model/src/test/derived/fieldset/test.sd @@ -0,0 +1,25 @@ +search test { + + document test { + + field word1 type array<string> { + indexing: index | summary | attribute + match: word + stemming: none + normalizing: none + } + + field word2 type array<string> { + indexing: index | summary | attribute + match: word + stemming: none + normalizing: none + } + + } + + fieldset words { + fields: word1,word2 + } + +}
\ No newline at end of file diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/FieldsetTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/FieldsetTestCase.java new file mode 100644 index 00000000000..6b3a3cf6a24 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/FieldsetTestCase.java @@ -0,0 +1,15 @@ +package com.yahoo.searchdefinition.derived; + +import com.yahoo.searchdefinition.parser.ParseException; +import org.junit.Test; + +import java.io.IOException; + +public class FieldsetTestCase extends AbstractExportingTestCase { + + @Test + public void testRankProfiles() throws IOException, ParseException { + assertCorrectDeriving("fieldset"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java index 3019b35cd2e..95503550767 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java @@ -87,6 +87,12 @@ public class NodeFlavorTuningTest { } @Test + public void require_that_search_read_mmap_advise_is_set_based_on_disk() { + assertSearchReadAdvise(ProtonConfig.Search.Mmap.Advise.RANDOM, true); + assertSearchReadAdvise(ProtonConfig.Search.Mmap.Advise.NORMAL, false); + } + + @Test public void require_that_summary_cache_max_bytes_is_set_based_on_memory() { assertEquals(1*GB/20, configFromMemorySetting(1).summary().cache().maxbytes()); assertEquals(256*GB/20, configFromMemorySetting(256).summary().cache().maxbytes()); @@ -115,6 +121,10 @@ public class NodeFlavorTuningTest { assertEquals(expValue, configFromDiskSetting(fastDisk).summary().read().io()); } + private static void assertSearchReadAdvise(ProtonConfig.Search.Mmap.Advise.Enum expValue, boolean fastDisk) { + assertEquals(expValue, configFromDiskSetting(fastDisk).search().mmap().advise()); + } + private static void assertSharedDisk(boolean sharedDisk, boolean docker) { assertEquals(sharedDisk, configFromEnvironmentType(docker).hwinfo().disk().shared()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java index 33099e3a36c..542d0b3f387 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/SuperModelManager.java @@ -64,7 +64,6 @@ public class SuperModelManager implements SuperModelProvider { return superModelConfigProvider.getSuperModel(); } - @Override public Zone getZone() { return zone; } diff --git a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java index a5e23bd45d3..484628397c0 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/ClustersStatus.java @@ -32,7 +32,7 @@ public class ClustersStatus extends AbstractComponent { private final Object mutex = new Object(); /** The status of clusters, when known. Note that clusters may exist for which there is no knowledge yet. */ - private final Map<Object, Boolean> clusterStatus = new HashMap<>(); + private final Map<String, Boolean> clusterStatus = new HashMap<>(); public void setContainerHasClusters(boolean containerHasClusters) { synchronized (mutex) { @@ -48,18 +48,34 @@ public class ClustersStatus extends AbstractComponent { } } - public void setUp(Object clusterIdentifier) { + void setUp(String clusterIdentifier) { synchronized (mutex) { clusterStatus.put(clusterIdentifier, Boolean.TRUE); } } - public void setDown(Object clusterIdentifier) { + void setDown(String clusterIdentifier) { synchronized (mutex) { clusterStatus.put(clusterIdentifier, Boolean.FALSE); } } + /** + @deprecated Use setUp(String) instead + */ + @Deprecated + public void setUp(Object clusterIdentifier) { + setUp((String) clusterIdentifier); + } + + /** + @deprecated Use setDown(String) instead + */ + @Deprecated + public void setDown(Object clusterIdentifier) { + setDown((String) clusterIdentifier); + } + /** Returns whether this container should receive traffic based on the state of this */ public boolean containerShouldReceiveTraffic() { synchronized (mutex) { diff --git a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java index be386b1b84e..ebb56af8853 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java +++ b/container-core/src/main/java/com/yahoo/container/handler/VipStatus.java @@ -1,9 +1,6 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.container.handler; -import java.util.IdentityHashMap; -import java.util.Map; - import com.google.inject.Inject; import com.yahoo.container.QrSearchersConfig; import com.yahoo.container.core.VipStatusConfig; @@ -52,15 +49,27 @@ public class VipStatus { } /** Note that a cluster (which influences up/down state) is up */ - public void addToRotation(Object clusterIdentifier) { + public void addToRotation(String clusterIdentifier) { clustersStatus.setUp(clusterIdentifier); } /** Note that a cluster (which influences up/down state) is down */ - public void removeFromRotation(Object clusterIdentifier) { + public void removeFromRotation(String clusterIdentifier) { clustersStatus.setDown(clusterIdentifier); } + /** @deprecated Use addToRotation(String) instead */ + @Deprecated + public void addToRotation(Object clusterIdentifier) { + addToRotation((String) clusterIdentifier); + } + + /** @deprecated Use removeFromRotation(String) instead */ + @Deprecated + public void removeFromRotation(Object clusterIdentifier) { + removeFromRotation((String) clusterIdentifier); + } + /** Returns whether this container should receive traffic at this time */ public boolean isInRotation() { if (inRotationOverride != null) return inRotationOverride; diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java index 1376b50dad9..e341b66e0f8 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/state/MetricsPacketsHandler.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import static com.yahoo.container.jdisc.state.StateHandler.getSnapshotPreprocessor; @@ -151,7 +152,7 @@ public class MetricsPacketsHandler extends AbstractRequestHandler { private void addMetaData(long timestamp, String application, JSONObjectWithLegibleException packet) { packet.put(APPLICATION_KEY, application); - packet.put(TIMESTAMP_KEY, timestamp); + packet.put(TIMESTAMP_KEY, TimeUnit.MILLISECONDS.toSeconds(timestamp)); } private void addDimensions(MetricDimensions metricDimensions, JSONObjectWithLegibleException packet) throws JSONException { diff --git a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java index e54f968f41d..4c1c1622140 100644 --- a/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java +++ b/container-core/src/test/java/com/yahoo/container/handler/VipStatusTestCase.java @@ -18,9 +18,9 @@ public class VipStatusTestCase { clustersStatus.setContainerHasClusters(true); VipStatus v = new VipStatus(clustersStatus); - Object cluster1 = new Object(); - Object cluster2 = new Object(); - Object cluster3 = new Object(); + String cluster1 = new String("a"); + String cluster2 = new String("b"); + String cluster3 = new String("c"); // initial state assertFalse(v.isInRotation()); diff --git a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java index fc914483117..92330345b50 100644 --- a/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java +++ b/container-core/src/test/java/com/yahoo/container/jdisc/state/MetricsPacketsHandlerTest.java @@ -17,6 +17,7 @@ import static com.yahoo.container.jdisc.state.MetricsPacketsHandler.PACKET_SEPAR import static com.yahoo.container.jdisc.state.MetricsPacketsHandler.STATUS_CODE_KEY; import static com.yahoo.container.jdisc.state.MetricsPacketsHandler.STATUS_MSG_KEY; import static com.yahoo.container.jdisc.state.MetricsPacketsHandler.TIMESTAMP_KEY; +import static com.yahoo.container.jdisc.state.StateHandlerTestBase.SNAPSHOT_INTERVAL; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -42,10 +43,7 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { @Test public void metrics_are_included_after_snapshot() throws Exception { metric.add("counter", 1, null); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); - String response = requestAsString("http://localhost/metrics-packets"); - - List<JsonNode> packets = toJsonPackets(response); + List<JsonNode> packets = incrementTimeAndGetJsonPackets(); assertEquals(2, packets.size()); JsonNode counterPacket = packets.get(1); @@ -55,10 +53,7 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { @Test public void metadata_is_included_in_each_metrics_packet() throws Exception { metric.add("counter", 1, null); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); - String response = requestAsString("http://localhost/metrics-packets"); - - List<JsonNode> packets = toJsonPackets(response); + List<JsonNode> packets = incrementTimeAndGetJsonPackets(); JsonNode counterPacket = packets.get(1); assertTrue(counterPacket.has(TIMESTAMP_KEY)); @@ -67,14 +62,22 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { } @Test + public void timestamp_resolution_is_in_seconds() throws Exception { + metric.add("counter", 1, null); + List<JsonNode> packets = incrementTimeAndGetJsonPackets(); + JsonNode counterPacket = packets.get(1); + + assertEquals(SNAPSHOT_INTERVAL/1000L, counterPacket.get(TIMESTAMP_KEY).asLong()); + } + + @Test public void expected_aggregators_are_output_for_gauge_metrics() throws Exception{ Metric.Context context = metric.createContext(Collections.singletonMap("dim1", "value1")); metric.set("gauge", 0.2, null); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); - String response = requestAsString("http://localhost/metrics-packets"); - List<JsonNode> packets = toJsonPackets(response); + List<JsonNode> packets = incrementTimeAndGetJsonPackets(); JsonNode gaugeMetric = packets.get(1).get(METRICS_KEY); + assertEquals(0.2, gaugeMetric.get("gauge.last").asDouble(), 0.1); assertEquals(0.2, gaugeMetric.get("gauge.average").asDouble(), 0.1); assertEquals(0.2, gaugeMetric.get("gauge.max").asDouble(), 0.1); @@ -84,10 +87,8 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { public void dimensions_from_context_are_included() throws Exception { Metric.Context context = metric.createContext(Collections.singletonMap("dim1", "value1")); metric.add("counter", 1, context); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); - String response = requestAsString("http://localhost/metrics-packets"); - List<JsonNode> packets = toJsonPackets(response); + List<JsonNode> packets = incrementTimeAndGetJsonPackets(); JsonNode counterPacket = packets.get(1); assertTrue(counterPacket.has(DIMENSIONS_KEY)); @@ -100,15 +101,12 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { Metric.Context context = metric.createContext(Collections.singletonMap("dim1", "value1")); metric.add("counter1", 1, context); metric.add("counter2", 2, context); - incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); - String response = requestAsString("http://localhost/metrics-packets"); - List<JsonNode> packets = toJsonPackets(response); + List<JsonNode> packets = incrementTimeAndGetJsonPackets(); assertEquals(2, packets.size()); JsonNode countersPacket = packets.get(1); assertEquals("value1", countersPacket.get(DIMENSIONS_KEY).get("dim1").asText()); - assertCountMetric(countersPacket, "counter1.count", 1); assertCountMetric(countersPacket, "counter2.count", 2); } @@ -119,11 +117,16 @@ public class MetricsPacketsHandlerTest extends StateHandlerTestBase { Metric.Context context2 = metric.createContext(Collections.singletonMap("dim2", "value2")); metric.add("counter1", 1, context1); metric.add("counter2", 1, context2); + + List<JsonNode> packets = incrementTimeAndGetJsonPackets(); + assertEquals(3, packets.size()); + } + + private List<JsonNode> incrementTimeAndGetJsonPackets() throws Exception { incrementCurrentTimeAndAssertSnapshot(SNAPSHOT_INTERVAL); String response = requestAsString("http://localhost/metrics-packets"); - List<JsonNode> packets = toJsonPackets(response); - assertEquals(3, packets.size()); + return toJsonPackets(response); } private List<JsonNode> toJsonPackets(String response) throws Exception { diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterMonitor.java index 4e708e32a2d..c075a0f842b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterMonitor.java @@ -100,9 +100,9 @@ public class ClusterMonitor implements Runnable, Freezable { if ( ! hasInformationAboutAllNodes()) return; if (hasWorkingNodesWithDocumentsOnline()) { - vipStatus.get().addToRotation(this); + vipStatus.get().addToRotation(nodeManager.getId().stringValue()); } else { - vipStatus.get().removeFromRotation(this); + vipStatus.get().removeFromRotation(nodeManager.getId().stringValue()); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 2fdcad32c32..3cb95fd0f7f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -110,7 +110,7 @@ public class ClusterSearcher extends Searcher { super(id); this.fs4ResourcePool = fs4ResourcePool; - Dispatcher dispatcher = new Dispatcher(dispatchConfig, fs4ResourcePool, clusterInfoConfig.nodeCount(), vipStatus); + Dispatcher dispatcher = new Dispatcher(id.stringValue(), dispatchConfig, fs4ResourcePool, clusterInfoConfig.nodeCount(), vipStatus); monitor = (dispatcher.searchCluster().directDispatchTarget().isPresent()) // dispatcher should decide vip status instead ? new ClusterMonitor(this, monitorConfig, Optional.empty()) diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index e95255b31e6..0382f47457e 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -49,22 +49,14 @@ public class Dispatcher extends AbstractComponent { private final RpcResourcePool rpcResourcePool; private final boolean multilevelDispatch; - public Dispatcher(DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { - this.searchCluster = new SearchCluster(dispatchConfig, fs4ResourcePool, containerClusterSize, vipStatus); + public Dispatcher(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { + this.searchCluster = new SearchCluster(clusterId, dispatchConfig, fs4ResourcePool, containerClusterSize, vipStatus); this.loadBalancer = new LoadBalancer(searchCluster, dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN); this.rpcResourcePool = new RpcResourcePool(dispatchConfig); this.multilevelDispatch = dispatchConfig.useMultilevelDispatch(); } - /** For testing */ - public Dispatcher(Map<Integer, Client.NodeConnection> nodeConnections, Client client) { - this.searchCluster = null; - this.loadBalancer = new LoadBalancer(searchCluster, true); - this.rpcResourcePool = new RpcResourcePool(client, nodeConnections); - this.multilevelDispatch = false; - } - /** Returns the search cluster this dispatches to */ public SearchCluster searchCluster() { return searchCluster; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index b51620a978b..b8d76906f70 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -41,6 +41,7 @@ public class SearchCluster implements NodeManager<Node> { private final double minGroupCoverage; private final int maxNodesDownPerGroup; private final int size; + private final String clusterId; private final ImmutableMap<Integer, Group> groups; private final ImmutableMultimap<String, Node> nodesByHost; private final ImmutableList<Group> orderedGroups; @@ -60,13 +61,14 @@ public class SearchCluster implements NodeManager<Node> { // Only needed until query requests are moved to rpc private final FS4ResourcePool fs4ResourcePool; - public SearchCluster(DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { - this(dispatchConfig.minActivedocsPercentage(), dispatchConfig.minGroupCoverage(), dispatchConfig.maxNodesDownPerGroup(), + public SearchCluster(String clusterId, DispatchConfig dispatchConfig, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { + this(clusterId, dispatchConfig.minActivedocsPercentage(), dispatchConfig.minGroupCoverage(), dispatchConfig.maxNodesDownPerGroup(), toNodes(dispatchConfig), fs4ResourcePool, containerClusterSize, vipStatus); } - public SearchCluster(double minActivedocsCoverage, double minGroupCoverage, int maxNodesDownPerGroup, List<Node> nodes, FS4ResourcePool fs4ResourcePool, + public SearchCluster(String clusterId, double minActivedocsCoverage, double minGroupCoverage, int maxNodesDownPerGroup, List<Node> nodes, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { + this.clusterId = clusterId; this.minActivedocsCoveragePercentage = minActivedocsCoverage; this.minGroupCoverage = minGroupCoverage; this.maxNodesDownPerGroup = maxNodesDownPerGroup; @@ -194,7 +196,7 @@ public class SearchCluster implements NodeManager<Node> { node.setWorking(true); if (usesDirectDispatchTo(node)) - vipStatus.addToRotation(this); + vipStatus.addToRotation(clusterId); } /** Used by the cluster monitor to manage node status */ @@ -204,16 +206,16 @@ public class SearchCluster implements NodeManager<Node> { // Take ourselves out if we usually dispatch only to our own host if (usesDirectDispatchTo(node)) - vipStatus.removeFromRotation(this); + vipStatus.removeFromRotation(clusterId); } private void updateSufficientCoverage(Group group, boolean sufficientCoverage) { // update VIP status if we direct dispatch to this group and coverage status changed if (usesDirectDispatchTo(group) && sufficientCoverage != group.hasSufficientCoverage()) { if (sufficientCoverage) { - vipStatus.addToRotation(this); + vipStatus.addToRotation(clusterId); } else { - vipStatus.removeFromRotation(this); + vipStatus.removeFromRotation(clusterId); } } group.setHasSufficientCoverage(sufficientCoverage); diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java index 79b43563c6a..3b8155efc95 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTestCase.java @@ -83,7 +83,7 @@ public class FastSearcherTestCase { Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); FastSearcher fastSearcher = new FastSearcher(new MockBackend(), new FS4ResourcePool(1), - new MockDispatcher(Collections.emptyList()), + new MockDispatcher("a", Collections.emptyList()), new SummaryParameters(null), new ClusterParams("testhittype"), new CacheParams(100, 1e64), @@ -104,7 +104,7 @@ public class FastSearcherTestCase { Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); FastSearcher fastSearcher = new FastSearcher(new MockBackend(), new FS4ResourcePool(1), - new MockDispatcher(Collections.emptyList()), + new MockDispatcher("a", Collections.emptyList()), new SummaryParameters(null), new ClusterParams("testhittype"), new CacheParams(100, 1e64), @@ -137,7 +137,7 @@ public class FastSearcherTestCase { MockFS4ResourcePool mockFs4ResourcePool = new MockFS4ResourcePool(); FastSearcher fastSearcher = new FastSearcher(new MockBackend(), mockFs4ResourcePool, - new MockDispatcher(nodes, mockFs4ResourcePool, 1, new VipStatus()), + new MockDispatcher("a", nodes, mockFs4ResourcePool, 1, new VipStatus()), new SummaryParameters(null), new ClusterParams("testhittype"), new CacheParams(0, 0), @@ -177,7 +177,7 @@ public class FastSearcherTestCase { new DocumentdbInfoConfig(new DocumentdbInfoConfig.Builder().documentdb(new DocumentdbInfoConfig.Documentdb.Builder().name("testDb"))); FastSearcher fastSearcher = new FastSearcher(mockBackend, new FS4ResourcePool(1), - new MockDispatcher(Collections.emptyList()), + new MockDispatcher("a", Collections.emptyList()), new SummaryParameters(null), new ClusterParams("testhittype"), new CacheParams(100, 1e64), @@ -369,7 +369,7 @@ public class FastSearcherTestCase { Logger.getLogger(FastSearcher.class.getName()).setLevel(Level.ALL); return new FastSearcher(mockBackend, new FS4ResourcePool(1), - new MockDispatcher(Collections.emptyList()), + new MockDispatcher("a", Collections.emptyList()), new SummaryParameters(null), new ClusterParams("testhittype"), new CacheParams(100, 1e64), @@ -382,7 +382,7 @@ public class FastSearcherTestCase { MockFSChannel.resetDocstamp(); FastSearcher fastSearcher = new FastSearcher(new MockBackend(), new FS4ResourcePool(1), - new MockDispatcher(Collections.emptyList()), + new MockDispatcher("a", Collections.emptyList()), new SummaryParameters(null), new ClusterParams("testhittype"), new CacheParams(100, 1e64), @@ -425,7 +425,7 @@ public class FastSearcherTestCase { MockFSChannel.resetDocstamp(); FastSearcher fastSearcher = new FastSearcher(new MockBackend(), new FS4ResourcePool(1), - new MockDispatcher(Collections.emptyList()), + new MockDispatcher("a", Collections.emptyList()), new SummaryParameters(null), new ClusterParams("testhittype"), new CacheParams(100, 1e64), @@ -479,7 +479,7 @@ public class FastSearcherTestCase { @Test public void testSinglePassGroupingIsNotForcedWithSingleNodeGroups() { MockDispatcher dispatcher = - new MockDispatcher(ImmutableList.of(new Node(0, "host0", 123, 0), + new MockDispatcher("a", ImmutableList.of(new Node(0, "host0", 123, 0), new Node(2, "host1", 123, 0))); FastSearcher fastSearcher = new FastSearcher(new MockBackend(), @@ -524,7 +524,7 @@ public class FastSearcherTestCase { Backend backend = listeners.getBackend(server.host.getHostString(),server.host.getPort()); FastSearcher fastSearcher = new FastSearcher(backend, new FS4ResourcePool(1), - new MockDispatcher(Collections.emptyList()), + new MockDispatcher("a", Collections.emptyList()), new SummaryParameters(null), new ClusterParams("testhittype"), new CacheParams(0, 0.0d), diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java index 12c313dbfe3..5377eb670c4 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/FastSearcherTester.java @@ -49,7 +49,7 @@ class FastSearcherTester { clustersStatus.setContainerHasClusters(true); vipStatus = new VipStatus(clustersStatus); mockFS4ResourcePool = new MockFS4ResourcePool(); - mockDispatcher = new MockDispatcher(searchNodes, mockFS4ResourcePool, containerClusterSize, vipStatus); + mockDispatcher = new MockDispatcher("a", searchNodes, mockFS4ResourcePool, containerClusterSize, vipStatus); fastSearcher = new FastSearcher(new MockBackend(selfHostname, 0L, true), mockFS4ResourcePool, mockDispatcher, diff --git a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java index 800b1bc21f0..1cd2c4ae791 100644 --- a/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java +++ b/container-search/src/test/java/com/yahoo/prelude/fastsearch/test/MockDispatcher.java @@ -14,20 +14,16 @@ import java.util.List; class MockDispatcher extends Dispatcher { public MockDispatcher(Node node) { - this(Collections.singletonList(node)); + this(node.hostname(), Collections.singletonList(node)); } - public MockDispatcher(List<Node> nodes) { - super(toDispatchConfig(nodes), new FS4ResourcePool(1), 1, new VipStatus()); + public MockDispatcher(String clusterId, List<Node> nodes) { + this(clusterId, nodes, new FS4ResourcePool(1), 1, new VipStatus()); } - public MockDispatcher(List<Node> nodes, VipStatus vipStatus) { - super(toDispatchConfig(nodes), new FS4ResourcePool(1), 1, vipStatus); - } - - public MockDispatcher(List<Node> nodes, FS4ResourcePool fs4ResourcePool, + public MockDispatcher(String clusterId, List<Node> nodes, FS4ResourcePool fs4ResourcePool, int containerClusterSize, VipStatus vipStatus) { - super(toDispatchConfig(nodes), fs4ResourcePool, containerClusterSize, vipStatus); + super(clusterId, toDispatchConfig(nodes), fs4ResourcePool, containerClusterSize, vipStatus); } private static DispatchConfig toDispatchConfig(List<Node> nodes) { diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 13b75cf9c5a..38a753360d8 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -22,7 +22,7 @@ public class LoadBalancerTest { @Test public void requreThatLoadBalancerServesSingleNodeSetups() { Node n1 = new Node(0, "test-node1", 0, 0); - SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1), null, 1, null); + SearchCluster cluster = new SearchCluster("a", 88.0, 99.0, 0, Arrays.asList(n1), null, 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroupForQuery(null); @@ -36,7 +36,7 @@ public class LoadBalancerTest { public void requreThatLoadBalancerServesMultiGroupSetups() { Node n1 = new Node(0, "test-node1", 0, 0); Node n2 = new Node(1, "test-node2", 1, 1); - SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null); + SearchCluster cluster = new SearchCluster("a", 88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroupForQuery(null); @@ -52,7 +52,7 @@ public class LoadBalancerTest { Node n2 = new Node(1, "test-node2", 1, 0); Node n3 = new Node(0, "test-node3", 0, 1); Node n4 = new Node(1, "test-node4", 1, 1); - SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2, n3, n4), null, 2, null); + SearchCluster cluster = new SearchCluster("a", 88.0, 99.0, 0, Arrays.asList(n1, n2, n3, n4), null, 2, null); LoadBalancer lb = new LoadBalancer(cluster, true); Optional<Group> grp = lb.takeGroupForQuery(null); @@ -63,7 +63,7 @@ public class LoadBalancerTest { public void requreThatLoadBalancerReturnsDifferentGroups() { Node n1 = new Node(0, "test-node1", 0, 0); Node n2 = new Node(1, "test-node2", 1, 1); - SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null); + SearchCluster cluster = new SearchCluster("a", 88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); // get first group @@ -83,7 +83,7 @@ public class LoadBalancerTest { public void requreThatLoadBalancerReturnsGroupWithShortestQueue() { Node n1 = new Node(0, "test-node1", 0, 0); Node n2 = new Node(1, "test-node2", 1, 1); - SearchCluster cluster = new SearchCluster(88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null); + SearchCluster cluster = new SearchCluster("a", 88.0, 99.0, 0, Arrays.asList(n1, n2), null, 1, null); LoadBalancer lb = new LoadBalancer(cluster, true); // get first group diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 0c0a65ded17..fc505097472 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -21,8 +21,8 @@ public class MockSearchCluster extends SearchCluster { private final ImmutableMap<Integer, Group> groups; private final ImmutableMultimap<String, Node> nodesByHost; - public MockSearchCluster(int groups, int nodesPerGroup) { - super(100, 100, 0, Collections.emptyList(), null, 1, null); + public MockSearchCluster(String clusterId, int groups, int nodesPerGroup) { + super(clusterId, 100, 100, 0, Collections.emptyList(), null, 1, null); ImmutableMap.Builder<Integer, Group> groupBuilder = ImmutableMap.builder(); ImmutableMultimap.Builder<String, Node> hostBuilder = ImmutableMultimap.builder(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/SearchPathTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/SearchPathTest.java index a1f926d3201..5a4457780e2 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/SearchPathTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/SearchPathTest.java @@ -69,7 +69,7 @@ public class SearchPathTest { @Test public void searchPathMustFilterNodesBasedOnDefinition() { - MockSearchCluster cluster = new MockSearchCluster(3, 3); + MockSearchCluster cluster = new MockSearchCluster("a",3, 3); assertThat(distKeysAsString(SearchPath.selectNodes("1/1", cluster)), equalTo("5")); assertThat(distKeysAsString(SearchPath.selectNodes("/1", cluster)), equalTo("4,5,6")); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 2705debf8ba..619f8abc180 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -151,6 +151,11 @@ public class Application { /** * Returns the oldest platform version this has deployed in a permanent zone (not test or staging). + * + * This is unfortunately quite similar to {@link ApplicationController#oldestInstalledPlatform(ApplicationId)}, + * but this checks only what the controller has deployed to the production zones, while that checks the node repository + * to see what's actually installed on each node. Thus, this is the right choice for, e.g., target Vespa versions for + * new deployments, while that is the right choice for version to compile against. */ public Optional<Version> oldestDeployedPlatform() { return productionDeployments().values().stream() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index ac317794725..953a226d089 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NoInstanceException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; @@ -67,6 +68,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -79,6 +81,10 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; +import static java.util.Comparator.naturalOrder; + /** * A singleton owned by the Controller which contains the methods and state for controlling applications. * @@ -164,6 +170,16 @@ public class ApplicationController { public ApplicationStore applicationStore() { return applicationStore; } + /** Returns the oldest Vespa version installed on any active or reserved production node for the given application. */ + public Version oldestInstalledPlatform(ApplicationId id) { + return get(id).flatMap(application -> application.productionDeployments().keySet().stream() + .flatMap(zone -> configServer().nodeRepository().list(zone, id, EnumSet.of(active, reserved)).stream()) + .map(Node::currentVersion) + .filter(version -> ! version.isEmpty()) + .min(naturalOrder())) + .orElse(controller.systemVersion()); + } + /** * Set the rotations marked as 'global' either 'in' or 'out of' service. * @@ -334,7 +350,7 @@ public class ApplicationController { } catch (RuntimeException e) { // If application has switched deployment pipeline, artifacts stored prior to the switch are in the other artifact store. log.info("Fetching application package for " + applicationId + " from alternate repository; it is now deployed " - + (application.get().deploymentJobs().deployedInternally() ? "internally" : "externally")); + + (application.get().deploymentJobs().deployedInternally() ? "internally" : "externally") + "\nException was: " + Exceptions.toMessageString(e)); applicationPackage = application.get().deploymentJobs().deployedInternally() ? new ApplicationPackage(artifactRepository.getApplicationPackage(application.get().id(), applicationVersion.id())) : new ApplicationPackage(applicationStore.get(application.get().id(), applicationVersion)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java index 2f56f2d9dd5..9e8b2495616 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java @@ -154,10 +154,15 @@ public class ApplicationList { return listOf(list.stream().filter(a -> a.deploymentSpec().canUpgradeAt(instant))); } - /** Returns the subset of applications that hasn't pinned to another major version than the given one */ - public ApplicationList allowMajorVersion(int majorVersion) { - return listOf(list.stream().filter(a -> ! a.deploymentSpec().majorVersion().isPresent() || - a.deploymentSpec().majorVersion().get().equals(majorVersion))); + /** + * Returns the subset of applications that hasn't pinned to an an earlier major version than the given one. + * + * @param targetMajorVersion the target major version which applications returned allows upgrading to + * @param defaultMajorVersion the default major version to assume for applications not specifying one + */ + public ApplicationList allowMajorVersion(int targetMajorVersion, int defaultMajorVersion) { + return listOf(list.stream().filter(a -> a.deploymentSpec().majorVersion().orElse(defaultMajorVersion) + >= targetMajorVersion)); } /** Returns the first n application in this (or all, if there are less than n). */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index d6182611fd5..8d3e1e9fffe 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -107,7 +107,7 @@ public class DeploymentTrigger { JobRun triggering; if (report.jobType() == component) { ApplicationVersion applicationVersion = ApplicationVersion.from(report.sourceRevision().get(), report.buildNumber()); - triggering = JobRun.triggering(application.get().oldestDeployedPlatform().orElse(controller.systemVersion()), applicationVersion, + triggering = JobRun.triggering(applications().oldestInstalledPlatform(report.applicationId()), applicationVersion, Optional.empty(), Optional.empty(), "Application commit", clock.instant()); if (report.success()) { if (acceptNewApplicationVersion(application.get())) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index e260848d93a..2e6b3d1360d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; +import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import com.yahoo.yolean.Exceptions; import java.time.Duration; @@ -51,7 +52,9 @@ public class ApplicationOwnershipConfirmer extends Maintainer { try { Tenant tenant = ownerOf(application.id()); Optional<IssueId> ourIssueId = application.ownershipIssueId(); - ourIssueId = ownershipIssues.confirmOwnership(ourIssueId, application.id(), userFor(tenant), tenant.contact().orElseThrow(RuntimeException::new)); + Contact contact = tenant.contact().orElseThrow(RuntimeException::new); + User asignee = tenant instanceof UserTenant ? userFor(tenant) : null; + ourIssueId = ownershipIssues.confirmOwnership(ourIssueId, application.id(), asignee, contact); ourIssueId.ifPresent(issueId -> store(issueId, application.id())); } catch (RuntimeException e) { // Catch errors due to wrong data in the controller, or issues client timeout. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 13733b32d86..3f5c2d1f317 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; +import com.yahoo.vespa.hosted.controller.tenant.UserTenant; import com.yahoo.yolean.Exceptions; import java.time.Duration; @@ -117,7 +118,7 @@ public class DeploymentIssueReporter extends Maintainer { private void fileDeploymentIssueFor(ApplicationId applicationId) { try { Tenant tenant = ownerOf(applicationId); - User asignee = userFor(tenant); + User asignee = tenant instanceof UserTenant ? userFor(tenant) : null; Optional<IssueId> ourIssueId = controller().applications().require(applicationId).deploymentJobs().issueId(); IssueId issueId = deploymentIssues.fileUnlessOpen(ourIssueId, applicationId, asignee, tenant.contact().get()); store(applicationId, issueId); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 66481e88a98..c1665dfba42 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -98,7 +98,7 @@ public class Upgrader extends Maintainer { private void upgrade(ApplicationList applications, Version version) { applications = applications.hasProductionDeployment(); applications = applications.onLowerVersionThan(version); - applications = applications.allowMajorVersion(version.getMajor()); + applications = applications.allowMajorVersion(version.getMajor(), targetMajorVersion().orElse(version.getMajor())); applications = applications.notDeploying(); // wait with applications deploying an application change or already upgrading applications = applications.notFailingOn(version); // try to upgrade only if it hasn't failed on this version applications = applications.canUpgradeAt(controller().clock().instant()); // wait with applications that are currently blocking upgrades @@ -127,9 +127,21 @@ public class Upgrader extends Maintainer { /** Sets the number of upgrades per minute */ public void setUpgradesPerMinute(double n) { + if (n < 0) + throw new IllegalArgumentException("Upgrades per minute must be >= 0, got " + n); curator.writeUpgradesPerMinute(n); } + /** Returns the target major version for applications not specifying one */ + public Optional<Integer> targetMajorVersion() { + return curator.readTargetMajorVersion(); + } + + /** Sets the default target major version. Set to empty to determine target version normally (by confidence) */ + public void setTargetMajorVersion(Optional<Integer> targetMajorVersion) { + curator.writeTargetMajorVersion(targetMajorVersion); + } + /** Override confidence for given version. This will cause the computed confidence to be ignored */ public void overrideConfidence(Version version, Confidence confidence) { try (Lock lock = curator.lockConfidenceOverrides()) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 9c52d7a2244..46432b7ca61 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -227,12 +227,20 @@ public class CuratorDb { } public void writeUpgradesPerMinute(double n) { - if (n < 0) { - throw new IllegalArgumentException("Upgrades per minute must be >= 0"); - } curator.set(upgradesPerMinutePath(), ByteBuffer.allocate(Double.BYTES).putDouble(n).array()); } - + + public Optional<Integer> readTargetMajorVersion() { + return read(targetMajorVersionPath(), ByteBuffer::wrap).map(ByteBuffer::getInt); + } + + public void writeTargetMajorVersion(Optional<Integer> targetMajorVersion) { + if (targetMajorVersion.isPresent()) + curator.set(targetMajorVersionPath(), ByteBuffer.allocate(Integer.BYTES).putInt(targetMajorVersion.get()).array()); + else + curator.delete(targetMajorVersionPath()); + } + public void writeVersionStatus(VersionStatus status) { curator.set(versionStatusPath(), asJson(versionStatusSerializer.toSlime(status))); } @@ -502,6 +510,10 @@ public class CuratorDb { return root.append("upgrader").append("upgradesPerMinute"); } + private static Path targetMajorVersionPath() { + return root.append("upgrader").append("targetMajorVersion"); + } + private static Path confidenceOverridesPath() { return root.append("upgrader").append("confidenceOverrides"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 3a83ecfb9e2..b3cba4809f0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -443,7 +443,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { }); // Compile version. The version that should be used when building an application - object.setString("compileVersion", application.oldestDeployedPlatform().orElse(controller.systemVersion()).toFullString()); + object.setString("compileVersion", controller.applications().oldestInstalledPlatform(application.id()).toFullString()); // Rotation Cursor globalRotationsArray = object.setArray("globalRotations"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index 19ec6316841..f6abaedd955 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -20,6 +20,7 @@ import com.yahoo.yolean.Exceptions; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; +import java.util.Optional; import java.util.Scanner; import java.util.logging.Level; @@ -102,7 +103,7 @@ public class ControllerApiHandler extends LoggingRequestHandler { private HttpResponse configureUpgrader(HttpRequest request) { String upgradesPerMinuteField = "upgradesPerMinute"; - String confidenceOverrideField = "confidenceOverride"; + String targetMajorVersionField = "targetMajorVersion"; byte[] jsonBytes = toJsonBytes(request.getData()); Inspector inspect = SlimeUtils.jsonToSlime(jsonBytes).get(); @@ -110,6 +111,9 @@ public class ControllerApiHandler extends LoggingRequestHandler { if (inspect.field(upgradesPerMinuteField).valid()) { upgrader.setUpgradesPerMinute(inspect.field(upgradesPerMinuteField).asDouble()); + } else if (inspect.field(targetMajorVersionField).valid()) { + int target = (int)inspect.field(targetMajorVersionField).asLong(); + upgrader.setTargetMajorVersion(Optional.ofNullable(target == 0 ? null : target)); // 0 is the default value } else { return ErrorResponse.badRequest("No such modifiable field(s)"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java index beb6c98e447..c4093c49fbd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java @@ -27,11 +27,14 @@ public class UpgraderResponse extends HttpResponse { Slime slime = new Slime(); Cursor root = slime.setObject(); root.setDouble("upgradesPerMinute", upgrader.upgradesPerMinute()); + upgrader.targetMajorVersion().ifPresent(v -> root.setLong("targetMajorVersion", v)); + Cursor array = root.setArray("confidenceOverrides"); upgrader.confidenceOverrides().forEach((version, confidence) -> { Cursor object = array.addObject(); object.setString(version.toString(), confidence.name()); }); + new JsonFormat(true).encode(outputStream, slime); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 587cbde991f..b81d9e78339 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -20,6 +20,7 @@ import org.junit.Test; import java.time.Duration; import java.time.Instant; import java.util.Collections; +import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; @@ -891,7 +892,7 @@ public class UpgraderTest { } @Test - public void testPinningMajorVersion() { + public void testPinningMajorVersionInApplication() { Version version = Version.fromString("6.2"); tester.upgradeSystem(version); @@ -925,6 +926,61 @@ public class UpgraderTest { } @Test + public void testPinningMajorVersionInUpgrader() { + Version version = Version.fromString("6.2"); + tester.upgradeSystem(version); + + ApplicationPackage version7CanaryApplicationPackage = new ApplicationPackageBuilder() + .majorVersion(7) + .upgradePolicy("canary") + .environment(Environment.prod) + .region("us-west-1") + .build(); + ApplicationPackage version7DefaultApplicationPackage = new ApplicationPackageBuilder() + .majorVersion(7) + .upgradePolicy("default") + .environment(Environment.prod) + .region("us-west-1") + .build(); + + // Setup applications + Application canary0 = tester.createAndDeploy("canary", 1, version7CanaryApplicationPackage); + Application default0 = tester.createAndDeploy("default0", 2, version7DefaultApplicationPackage); + Application default1 = tester.createAndDeploy("default1", 3, "default"); + + // New major version is released, but we don't want to upgrade to it yet + tester.upgrader().setTargetMajorVersion(Optional.of(6)); + version = Version.fromString("7.0"); + tester.upgradeSystem(version); + assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber()); + tester.triggerUntilQuiescence(); + + // ... canary upgrade to it because it explicitly wants 7 + assertEquals(2, tester.buildService().jobs().size()); + tester.completeUpgrade(canary0, version, version7CanaryApplicationPackage); + assertEquals(0, tester.buildService().jobs().size()); + tester.computeVersionStatus(); + + // default0 upgrades, but not default1 + tester.upgrader().maintain(); + tester.triggerUntilQuiescence(); + assertEquals(2, tester.buildService().jobs().size()); + tester.completeUpgrade(default0, version, version7DefaultApplicationPackage); + + // Nothing more happens ... + tester.upgrader().maintain(); + tester.triggerUntilQuiescence(); + assertEquals(0, tester.buildService().jobs().size()); + + // Now we want upgrade the latest application + tester.upgrader().setTargetMajorVersion(Optional.empty()); + tester.upgrader().maintain(); + tester.triggerUntilQuiescence(); + assertEquals(2, tester.buildService().jobs().size()); + tester.completeUpgrade(default1, version, "default"); + } + + @Test public void testAllowApplicationChangeDuringFailingUpgrade() { Version version = Version.fromString("6.2"); tester.upgradeSystem(version); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index e5f3af8c06a..19e04a617f6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -19,7 +19,7 @@ public class ControllerApiTest extends ControllerContainerTest { private static final AthenzIdentity HOSTED_VESPA_OPERATOR = AthenzUser.fromUserId("johnoperator"); @Test - public void testControllerApi() throws Exception { + public void testControllerApi() { ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/", new byte[0], Request.Method.GET), new File("root.json")); @@ -40,7 +40,7 @@ public class ControllerApiTest extends ControllerContainerTest { } @Test - public void testUpgraderApi() throws Exception { + public void testUpgraderApi() { addUserToHostedOperatorRole(HOSTED_VESPA_OPERATOR); ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); @@ -53,7 +53,7 @@ public class ControllerApiTest extends ControllerContainerTest { // Set invalid configuration tester.assertResponse( hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"upgradesPerMinute\":-1}", Request.Method.PATCH), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Upgrades per minute must be >= 0\"}", + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Upgrades per minute must be >= 0, got -1.0\"}", 400); // Ignores unrecognized field @@ -68,6 +68,18 @@ public class ControllerApiTest extends ControllerContainerTest { "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[]}", 200); + // Set target major version + tester.assertResponse( + hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"targetMajorVersion\":6}", Request.Method.PATCH), + "{\"upgradesPerMinute\":42.0,\"targetMajorVersion\":6,\"confidenceOverrides\":[]}", + 200); + + // Clear target major version + tester.assertResponse( + hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader", "{\"targetMajorVersion\":null}", Request.Method.PATCH), + "{\"upgradesPerMinute\":42.0,\"confidenceOverrides\":[]}", + 200); + // Override confidence tester.assertResponse( hostedOperatorRequest("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42", "broken", Request.Method.POST), diff --git a/fastos/src/vespa/fastos/file.h b/fastos/src/vespa/fastos/file.h index a9be683d395..8424967f655 100644 --- a/fastos/src/vespa/fastos/file.h +++ b/fastos/src/vespa/fastos/file.h @@ -103,7 +103,6 @@ protected: static FailedHandler _failedHandler; public: - static int getDefaultFAdviseOptions() { return _defaultFAdviseOptions; } static void setDefaultFAdviseOptions(int options) { _defaultFAdviseOptions = options; } int getFAdviseOptions() const { return _fAdviseOptions; } void setFAdviseOptions(int options) { _fAdviseOptions = options; } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index faa28d745fe..9d9ec9bc9b3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -120,7 +120,7 @@ public class StorageMaintainer { .withTags(tags)); // configserver-new - Path configServerNewCheckPath = Paths.get("curl"); + Path configServerNewCheckPath = Paths.get("/usr/bin/curl"); configs.add(new SecretAgentCheckConfig(nodeTypeToRole(context.nodeType())+"-new", 60, configServerNewCheckPath, "-s", "localhost:19071/yamas-metrics") .withTags(tags)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 6b9a7d4021a..25ed3ec4b59 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -626,6 +626,7 @@ public class NodeAgentImpl implements NodeAgent { long memoryTotalBytesUsed = memoryTotalBytesUsage - memoryTotalBytesCache; double memoryUsageRatio = (double) memoryTotalBytesUsed / memoryTotalBytes; + double memoryTotalUsageRatio = (double) memoryTotalBytesUsage / memoryTotalBytes; Optional<Double> diskUsageRatio = diskTotalBytesUsed.map(used -> (double) used / diskTotalBytes); List<DimensionMetrics> metrics = new ArrayList<>(); @@ -633,6 +634,8 @@ public class NodeAgentImpl implements NodeAgent { .withMetric("mem.limit", memoryTotalBytes) .withMetric("mem.used", memoryTotalBytesUsed) .withMetric("mem.util", 100 * memoryUsageRatio) + .withMetric("mem_total.used", memoryTotalBytesUsage) + .withMetric("mem_total.util", 100 * memoryTotalUsageRatio) .withMetric("cpu.util", 100 * cpuUsageRatioOfAllocated) .withMetric("cpu.sys.util", 100 * cpuKernelUsageRatioOfAllocated) .withMetric("disk.limit", diskTotalBytes); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 83ee9b57918..b6128fc8693 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -655,7 +655,9 @@ public class NodeAgentImplTest { long calledTimeout = (long) invocation.getArguments()[1]; String[] calledCommand = new String[invocation.getArguments().length - 2]; System.arraycopy(invocation.getArguments(), 2, calledCommand, 0, calledCommand.length); - calledCommand[calledCommand.length - 1] = calledCommand[calledCommand.length - 1].replaceAll("\"timestamp\":\\d+", "\"timestamp\":0"); + calledCommand[calledCommand.length - 1] = calledCommand[calledCommand.length - 1] + .replaceAll("\"timestamp\":\\d+", "\"timestamp\":0") + .replaceAll("([0-9]+\\.[0-9]{1,3})([0-9]*)", "$1"); // Only keep the first 3 decimals assertEquals(context, calledContainerName); assertEquals(5L, calledTimeout); diff --git a/node-admin/src/test/resources/expected.container.system.metrics.txt b/node-admin/src/test/resources/expected.container.system.metrics.txt index fab0f822d19..25de20803c7 100644 --- a/node-admin/src/test/resources/expected.container.system.metrics.txt +++ b/node-admin/src/test/resources/expected.container.system.metrics.txt @@ -10,6 +10,8 @@ s: "metrics": { "mem.limit": 4294967296, "mem.used": 1073741824, + "mem_total.util": 40.808, + "mem_total.used": 1752707072, "disk.used": 39625000000, "cpu.sys.util": 3.402, "disk.util": 15.85, diff --git a/parent/pom.xml b/parent/pom.xml index 4f4fa64dc31..3a0d568a878 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -678,6 +678,16 @@ <artifactId>jna</artifactId> <version>${jna.version}</version> </dependency> + <dependency> + <groupId>org.tensorflow</groupId> + <artifactId>proto</artifactId> + <version>${tensorflow.version}</version> + </dependency> + <dependency> + <groupId>org.tensorflow</groupId> + <artifactId>tensorflow</artifactId> + <version>${tensorflow.version}</version> + </dependency> </dependencies> </dependencyManagement> @@ -686,6 +696,7 @@ <antlr4.version>4.5</antlr4.version> <asm.version>6.2</asm.version> <jna.version>4.5.2</jna.version> + <tensorflow.version>1.11.0</tensorflow.version> <!-- Athenz dependencies. Make sure these dependencies matches those in Vespa's internal repositories --> <athenz.version>1.7.43</athenz.version> <commons-lang.version>2.6</commons-lang.version> diff --git a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp index d1d5b1c9af7..7a31580fb16 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/exclusive_attribute_read_accessor.cpp @@ -37,8 +37,9 @@ ExclusiveAttributeReadAccessor(const AttributeVector::SP &attribute, namespace { void -attributeWriteBlockingTask(GateSP entranceGate, GateSP exitGate) +attributeWriteBlockingTask(AttributeVector::SP attribute, GateSP entranceGate, GateSP exitGate) { + attribute->commit(true); entranceGate->countDown(); exitGate->await(); } @@ -51,7 +52,7 @@ ExclusiveAttributeReadAccessor::takeGuard() GateSP entranceGate = std::make_shared<Gate>(); GateSP exitGate = std::make_shared<Gate>(); _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorId(_attribute->getNamePrefix()), - [entranceGate, exitGate]() { attributeWriteBlockingTask(entranceGate, exitGate); }); + [this, entranceGate, exitGate]() { attributeWriteBlockingTask(_attribute, entranceGate, exitGate); }); entranceGate->await(); return std::make_unique<Guard>(*_attribute, exitGate); } diff --git a/searchlib/pom.xml b/searchlib/pom.xml index 3d5d8ee8e20..f1be4e96269 100644 --- a/searchlib/pom.xml +++ b/searchlib/pom.xml @@ -41,12 +41,10 @@ <dependency> <groupId>org.tensorflow</groupId> <artifactId>proto</artifactId> - <version>1.11.0</version> </dependency> <dependency> <groupId>org.tensorflow</groupId> <artifactId>tensorflow</artifactId> - <version>1.11.0</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> diff --git a/searchlib/src/vespa/searchlib/common/tunefileinfo.h b/searchlib/src/vespa/searchlib/common/tunefileinfo.h index edd6d29dd58..d431a9cf78f 100644 --- a/searchlib/src/vespa/searchlib/common/tunefileinfo.h +++ b/searchlib/src/vespa/searchlib/common/tunefileinfo.h @@ -20,7 +20,6 @@ private: public: TuneFileSeqRead() : _tuneControl(NORMAL) { } - void setWantNormal() { _tuneControl = NORMAL; } void setWantDirectIO() { _tuneControl = DIRECTIO; } bool getWantDirectIO() const { return _tuneControl == DIRECTIO; } @@ -64,8 +63,6 @@ private: public: TuneFileSeqWrite() : _tuneControl(NORMAL) { } - void setWantNormal() { _tuneControl = NORMAL; } - void setWantSyncWrites() { _tuneControl = OSYNC; } void setWantDirectIO() { _tuneControl = DIRECTIO; } bool getWantDirectIO() const { return _tuneControl == DIRECTIO; } bool getWantSyncWrites() const { return _tuneControl == OSYNC; } @@ -108,7 +105,6 @@ public: _advise(0) { } - void setMemoryMapFlags(int flags) { _mmapFlags = flags; } void setAdvise(int advise) { _advise = advise; } void setWantMemoryMap() { _tuneControl = MMAP; } void setWantDirectIO() { _tuneControl = DIRECTIO; } diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.cpp index 56a42eb161b..b898405be07 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.cpp @@ -7,9 +7,7 @@ #include <vespa/log/log.h> LOG_SETUP(".diskindex.bitvectordictionary"); -namespace search { - -namespace diskindex { +namespace search::diskindex { BitVectorDictionary::BitVectorDictionary() @@ -23,7 +21,7 @@ BitVectorDictionary::BitVectorDictionary() BitVectorDictionary::~BitVectorDictionary() { - if (_datFile.get() != NULL) { + if (_datFile) { _datFile->Close(); } } @@ -64,7 +62,9 @@ BitVectorDictionary::open(const vespalib::string &pathPrefix, idxFile.Close(); _vectorSize = BitVector::getFileBytes(_docIdLimit); - _datFile.reset(new FastOS_File()); + _datFile = std::make_unique<FastOS_File>(); + _datFile->setFAdviseOptions(tuneFileRead.getAdvise()); + if (tuneFileRead.getWantMemoryMap()) { _datFile->enableMemoryMap(tuneFileRead.getMemoryMapFlags()); } else if (tuneFileRead.getWantDirectIO()) { @@ -100,7 +100,4 @@ BitVectorDictionary::lookup(uint64_t wordNum) itr->_numDocs); } - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h index fa3c51d737e..79c892c6ed7 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h @@ -7,9 +7,7 @@ #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/vespalib/stllike/string.h> -namespace search { - -namespace diskindex { +namespace search::diskindex { /** * This dictionary provides a sparse mapping from word number -> BitVector. @@ -53,7 +51,7 @@ public: * bit vector if found. * * @param wordNum the word number to lookup a bit vector for. - * @return the loaded bit vector or NULL if not found. + * @return the loaded bit vector or nullptr if not found. **/ BitVector::UP lookup(uint64_t wordNum); @@ -62,7 +60,5 @@ public: const std::vector<WordSingleKey> & getEntries() const { return _entries; } }; -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp index 374c6a8c7e6..fa924895342 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.cpp @@ -29,7 +29,7 @@ const size_t FILE_HEADERSIZE_ALIGNMENT = 4096; BitVectorFileWrite::BitVectorFileWrite(BitVectorKeyScope scope) : BitVectorIdxFileWrite(scope), - _datFile(NULL), + _datFile(nullptr), _datHeaderLen(0) { } @@ -50,7 +50,7 @@ BitVectorFileWrite::open(const vespalib::string &name, { vespalib::string datname = name + ".bdat"; - assert(_datFile == NULL); + assert(_datFile == nullptr); Parent::open(name, docIdLimit, tuneFileWrite, fileHeaderContext); @@ -159,7 +159,7 @@ BitVectorFileWrite::close() { size_t bitmapbytes = BitVector::getFileBytes(_docIdLimit); - if (_datFile != NULL) { + if (_datFile != nullptr) { if (_datFile->IsOpened()) { uint64_t pos = _datFile->GetPosition(); assert(pos == static_cast<uint64_t>(_numKeys) * @@ -170,12 +170,11 @@ BitVectorFileWrite::close() _datFile->Close(); } delete _datFile; - _datFile = NULL; + _datFile = nullptr; } Parent::close(); } -BitVectorCandidate::~BitVectorCandidate() { -} +BitVectorCandidate::~BitVectorCandidate() = default; } diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h index 893b792d96b..071cf37443d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h @@ -8,22 +8,12 @@ #include <vespa/vespalib/stllike/string.h> #include "bitvectoridxfile.h" -namespace search -{ - - -namespace diskindex -{ +namespace search::diskindex { class BitVectorFileWrite : public BitVectorIdxFileWrite { private: - BitVectorFileWrite(const BitVectorFileWrite &) = delete; - BitVectorFileWrite(const BitVectorFileWrite &&) = delete; - BitVectorFileWrite& operator=(const BitVectorFileWrite &) = delete; - BitVectorFileWrite& operator=(const BitVectorFileWrite &&) = delete; - using Parent = BitVectorIdxFileWrite; Fast_BufferedFile *_datFile; @@ -33,33 +23,24 @@ private: uint32_t _datHeaderLen; public: + BitVectorFileWrite(const BitVectorFileWrite &) = delete; + BitVectorFileWrite(const BitVectorFileWrite &&) = delete; + BitVectorFileWrite& operator=(const BitVectorFileWrite &) = delete; + BitVectorFileWrite& operator=(const BitVectorFileWrite &&) = delete; BitVectorFileWrite(BitVectorKeyScope scope); - ~BitVectorFileWrite(); - void - open(const vespalib::string &name, uint32_t docIdLimit, - const TuneFileSeqWrite &tuneFileWrite, - const common::FileHeaderContext &fileHeaderContext); - - - void - addWordSingle(uint64_t wordNum, const BitVector &bitVector); - - void - flush(); - - void - sync(); - - void - close(); + void open(const vespalib::string &name, uint32_t docIdLimit, + const TuneFileSeqWrite &tuneFileWrite, + const common::FileHeaderContext &fileHeaderContext); - void - makeDatHeader(const common::FileHeaderContext &fileHeaderContext); - void - updateDatHeader(uint64_t fileBitSize); + void addWordSingle(uint64_t wordNum, const BitVector &bitVector); + void flush(); + void sync(); + void close(); + void makeDatHeader(const common::FileHeaderContext &fileHeaderContext); + void updateDatHeader(uint64_t fileBitSize); }; @@ -86,19 +67,12 @@ public: BitVectorCandidate(uint32_t docIdLimit) - : _array(), - _numDocs(0u), - _bitVectorLimit(BitVectorFileWrite::getBitVectorLimit(docIdLimit)), - _bv(BitVector::create(docIdLimit)) - { - _array.reserve(_bitVectorLimit); - } + : BitVectorCandidate(docIdLimit, BitVectorFileWrite::getBitVectorLimit(docIdLimit)) + { } ~BitVectorCandidate(); - void - clear() - { + void clear() { if (__builtin_expect(_numDocs > _bitVectorLimit, false)) { _bv->clear(); } @@ -106,9 +80,7 @@ public: _array.clear(); } - void - flush(BitVector &obv) - { + void flush(BitVector &obv) { if (__builtin_expect(_numDocs > _bitVectorLimit, false)) { obv.orWith(*_bv); } else { @@ -119,9 +91,7 @@ public: clear(); } - void - add(uint32_t docId) - { + void add(uint32_t docId) { if (_numDocs < _bitVectorLimit) { _array.push_back(docId); } else { @@ -139,37 +109,19 @@ public: /* * Get number of documents buffered. This might include duplicates. */ - uint64_t - getNumDocs() const - { - return _numDocs; - } + uint64_t getNumDocs() const { return _numDocs; } - bool - empty() const - { - return _numDocs == 0; - } + bool empty() const { return _numDocs == 0; } /* * Return true if array limit has been exceeded and bitvector has been * populated. */ - bool - getCrossedBitVectorLimit() const - { + bool getCrossedBitVectorLimit() const { return _numDocs > _bitVectorLimit; } - BitVector & - getBitVector() - { - return *_bv; - } + BitVector &getBitVector() { return *_bv; } }; - -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp index a4eca7feac9..3d7550c9c26 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.cpp @@ -14,8 +14,7 @@ using search::common::FileHeaderContext; namespace { void -readHeader(vespalib::FileHeader &h, - const vespalib::string &name) +readHeader(vespalib::FileHeader &h, const vespalib::string &name) { Fast_BufferedFile file(32768u); file.OpenReadOnly(name.c_str()); @@ -28,7 +27,7 @@ const size_t FILE_HEADERSIZE_ALIGNMENT = 4096; } BitVectorIdxFileWrite::BitVectorIdxFileWrite(BitVectorKeyScope scope) - : _idxFile(NULL), + : _idxFile(), _numKeys(0), _docIdLimit(0), _idxHeaderLen(0), @@ -37,11 +36,7 @@ BitVectorIdxFileWrite::BitVectorIdxFileWrite(BitVectorKeyScope scope) } -BitVectorIdxFileWrite::~BitVectorIdxFileWrite() -{ - // No implicit close() call, but cleanup memory allocations. - delete _idxFile; -} +BitVectorIdxFileWrite::~BitVectorIdxFileWrite() = default; uint64_t @@ -65,9 +60,8 @@ BitVectorIdxFileWrite::open(const vespalib::string &name, } vespalib::string idxname = name + getBitVectorKeyScopeSuffix(_scope); - assert(_idxFile == NULL); - FastOS_FileInterface *idxfile = new FastOS_File; - _idxFile = new Fast_BufferedFile(idxfile); + assert( !_idxFile); + _idxFile = std::make_unique<Fast_BufferedFile>(new FastOS_File()); if (tuneFileWrite.getWantSyncWrites()) _idxFile->EnableSyncWrites(); if (tuneFileWrite.getWantDirectIO()) @@ -174,7 +168,7 @@ BitVectorIdxFileWrite::sync() void BitVectorIdxFileWrite::close() { - if (_idxFile != NULL) { + if (_idxFile) { if (_idxFile->IsOpened()) { uint64_t pos = _idxFile->GetPosition(); assert(pos == idxSize()); @@ -182,8 +176,7 @@ BitVectorIdxFileWrite::close() updateIdxHeader(pos * 8); _idxFile->Close(); } - delete _idxFile; - _idxFile = NULL; + _idxFile.reset(); } } diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h index 8d704d1bffe..47894764976 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectoridxfile.h @@ -8,30 +8,15 @@ #include <vespa/vespalib/stllike/string.h> #include "bitvectorkeyscope.h" -namespace search -{ - +namespace search::common { class FileHeaderContext; } -namespace common -{ -class FileHeaderContext; - -} - - -namespace diskindex -{ +namespace search::diskindex { class BitVectorIdxFileWrite { private: - BitVectorIdxFileWrite(const BitVectorIdxFileWrite &) = delete; - BitVectorIdxFileWrite(const BitVectorIdxFileWrite &&) = delete; - BitVectorIdxFileWrite& operator=(const BitVectorIdxFileWrite &) = delete; - BitVectorIdxFileWrite& operator=(const BitVectorIdxFileWrite &&) = delete; - - Fast_BufferedFile *_idxFile; + std::unique_ptr<Fast_BufferedFile> _idxFile; public: @@ -45,32 +30,26 @@ protected: void syncCommon(); public: + BitVectorIdxFileWrite(const BitVectorIdxFileWrite &) = delete; + BitVectorIdxFileWrite(const BitVectorIdxFileWrite &&) = delete; + BitVectorIdxFileWrite& operator=(const BitVectorIdxFileWrite &) = delete; + BitVectorIdxFileWrite& operator=(const BitVectorIdxFileWrite &&) = delete; BitVectorIdxFileWrite(BitVectorKeyScope scope); ~BitVectorIdxFileWrite(); - void - open(const vespalib::string &name, uint32_t docIdLimit, - const TuneFileSeqWrite &tuneFileWrite, - const search::common::FileHeaderContext &fileHeaderContext); - - + void open(const vespalib::string &name, uint32_t docIdLimit, + const TuneFileSeqWrite &tuneFileWrite, + const common::FileHeaderContext &fileHeaderContext); - void - addWordSingle(uint64_t wordNum, uint32_t numDocs); - void - flush(); - void - sync(); + void addWordSingle(uint64_t wordNum, uint32_t numDocs); + void flush(); + void sync(); + void close(); - void - close(); - - static uint32_t - getBitVectorLimit(uint32_t docIdLimit) - { + static uint32_t getBitVectorLimit(uint32_t docIdLimit) { // Must match FastS_BinSizeParams::CalcMaxBinSize() uint32_t ret = (docIdLimit + 63) / 64; if (ret < 16) @@ -80,15 +59,8 @@ public: return ret; } - void - makeIdxHeader(const search::common::FileHeaderContext &fileHeaderContext); - - void - updateIdxHeader(uint64_t fileBitSize); + void makeIdxHeader(const common::FileHeaderContext &fileHeaderContext); + void updateIdxHeader(uint64_t fileBitSize); }; - -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.h b/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.h index 7b2af6adb9d..99deae5542b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorkeyscope.h @@ -2,12 +2,7 @@ #pragma once - -namespace search -{ - -namespace diskindex -{ +namespace search::diskindex { enum class BitVectorKeyScope { @@ -18,5 +13,3 @@ enum class BitVectorKeyScope const char *getBitVectorKeyScopeSuffix(BitVectorKeyScope scope); } - -} diff --git a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp index b820a77724b..22cd42da06f 100644 --- a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.cpp @@ -22,9 +22,7 @@ DictionaryWordReader::DictionaryWordReader() } -DictionaryWordReader::~DictionaryWordReader() -{ -} +DictionaryWordReader::~DictionaryWordReader() = default; bool diff --git a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h index ce791314b36..9c8db85bbeb 100644 --- a/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h +++ b/searchlib/src/vespa/searchlib/diskindex/dictionarywordreader.h @@ -4,12 +4,7 @@ #include "pagedict4file.h" #include <vespa/fastlib/io/bufferedfile.h> -namespace search -{ - -namespace diskindex -{ - +namespace search::diskindex { /* * Helper class, will be used by fusion later to handle generation of @@ -28,20 +23,14 @@ public: { } - void - tryWriteWord(vespalib::stringref word) - { + void tryWriteWord(vespalib::stringref word) { if (word != _word || _wordNum == 0) { ++_wordNum; _word = word; } } - uint64_t - getWordNum() const - { - return _wordNum; - } + uint64_t getWordNum() const { return _wordNum; } }; @@ -63,35 +52,17 @@ private: using DictionaryFileSeqRead = index::DictionaryFileSeqRead; std::unique_ptr<DictionaryFileSeqRead> _dictFile; - void - allocFiles(); - - static uint64_t - noWordNumHigh() - { - return std::numeric_limits<uint64_t>::max(); - } - - static uint64_t - noWordNum() - { - return 0u; - } - + static uint64_t noWordNumHigh() { return std::numeric_limits<uint64_t>::max(); } + static uint64_t noWordNum() { return 0u; } public: DictionaryWordReader(); - ~DictionaryWordReader(); - bool - isValid() const - { + bool isValid() const { return _wordNum != noWordNumHigh(); } - bool - operator<(const DictionaryWordReader &rhs) const - { + bool operator<(const DictionaryWordReader &rhs) const { if (!isValid()) return false; if (!rhs.isValid()) @@ -99,37 +70,24 @@ public: return _word < rhs._word; } - void - read() - { + void read() { _dictFile->readWord(_word, _wordNum, _counts); } - bool - open(const vespalib::string & dictionaryName, - const vespalib::string & wordMapName, - const TuneFileSeqRead &tuneFileRead); + bool open(const vespalib::string & dictionaryName, + const vespalib::string & wordMapName, + const TuneFileSeqRead &tuneFileRead); - void - close(); + void close(); - void - writeNewWordNum(uint64_t newWordNum) - { + void writeNewWordNum(uint64_t newWordNum) { _old2newwordfile->WriteBuf(&newWordNum, sizeof(newWordNum)); } - void - write(WordAggregator &writer) - { + void write(WordAggregator &writer) { writer.tryWriteWord(_word); writeNewWordNum(writer.getWordNum()); } }; - - -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp b/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp index e2fb7367b94..9a6144e9635 100644 --- a/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/diskindex.cpp @@ -20,9 +20,7 @@ using namespace search::index; using namespace search::query; using namespace search::queryeval; -namespace search { - -namespace diskindex { +namespace search::diskindex { void swap(DiskIndex::LookupResult & a, DiskIndex::LookupResult & b) { @@ -37,12 +35,12 @@ DiskIndex::LookupResult::LookupResult() { } -DiskIndex::Key::Key() : _indexes() { } +DiskIndex::Key::Key() = default; DiskIndex::Key::Key(const IndexList & indexes, vespalib::stringref word) : _word(word), _indexes(indexes) { } -DiskIndex::Key::~Key() { } +DiskIndex::Key::~Key() = default; DiskIndex::DiskIndex(const vespalib::string &indexDir, size_t cacheSize) : _indexDir(indexDir), @@ -58,7 +56,7 @@ DiskIndex::DiskIndex(const vespalib::string &indexDir, size_t cacheSize) calculateSize(); } -DiskIndex::~DiskIndex() {} +DiskIndex::~DiskIndex() = default; bool DiskIndex::loadSchema() @@ -301,7 +299,7 @@ DiskIndex::readPostingList(const LookupResult &lookupRes) const handle->_bitLength = lookupRes.counts._bitLength; SchemaUtil::IndexIterator it(_schema, lookupRes.indexId); handle->_file = _postingFiles[it.getIndex()].get(); - if (handle->_file == NULL) { + if (handle->_file == nullptr) { return PostingListHandle::UP(); } const uint32_t firstSegment = 0; @@ -319,7 +317,7 @@ DiskIndex::readBitVector(const LookupResult &lookupRes) const { SchemaUtil::IndexIterator it(_schema, lookupRes.indexId); BitVectorDictionary * dict = _bitVectorDicts[it.getIndex()].get(); - if (dict == NULL) { + if (dict == nullptr) { return BitVector::UP(); } return dict->lookup(lookupRes.wordNum); @@ -471,7 +469,4 @@ DiskIndex::createBlueprint(const IRequestContext & requestContext, const FieldSp } } - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/diskindex.h b/searchlib/src/vespa/searchlib/diskindex/diskindex.h index dd3daab34cd..4bef53a3030 100644 --- a/searchlib/src/vespa/searchlib/diskindex/diskindex.h +++ b/searchlib/src/vespa/searchlib/diskindex/diskindex.h @@ -9,10 +9,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/cache.h> - -namespace search { - -namespace diskindex { +namespace search::diskindex { /** * This class represents a disk index with a common dictionary, and @@ -109,7 +106,7 @@ public: * @param indexId the id of the field to * perform lookup for. * @param word the word to lookup. - * @return the lookup result or NULL if the word is not found. + * @return the lookup result or nullptr if the word is not found. **/ LookupResult::UP lookup(uint32_t indexId, vespalib::stringref word); LookupResultVector lookup(const std::vector<uint32_t> & indexes, vespalib::stringref word); @@ -127,7 +124,7 @@ public: * Read the bit vector corresponding to the given lookup result. * * @param lookupRes the result of the previous dictionary lookup. - * @return the bit vector or NULL if no bit vector exists for the + * @return the bit vector or nullptr if no bit vector exists for the * word in the lookup result. **/ BitVector::UP readBitVector(const LookupResult &lookupRes) const; @@ -150,7 +147,6 @@ public: const index::Schema &getSchema() const { return _schema; } const vespalib::string &getIndexDir() const { return _indexDir; } - const TuneFileSearch &getTuneFileSearch() const { return _tuneFileSearch; } /** * Needed for the Cache::BackingStore interface. @@ -160,6 +156,4 @@ public: void swap(DiskIndex::LookupResult & a, DiskIndex::LookupResult & b); -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp index e2c3dafc13d..4c159fe6114 100644 --- a/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/disktermblueprint.cpp @@ -56,9 +56,9 @@ namespace { bool areAnyParentsEquiv(const Blueprint * node) { - return (node == NULL) + return (node == nullptr) ? false - : (dynamic_cast<const EquivBlueprint *>(node) != NULL) + : (dynamic_cast<const EquivBlueprint *>(node) != nullptr) ? true : areAnyParentsEquiv(node->getParent()); } @@ -71,7 +71,7 @@ DiskTermBlueprint::fetchPostings(bool strict) (void) strict; _hasEquivParent = areAnyParentsEquiv(getParent()); _bitVector = _diskIndex.readBitVector(*_lookupRes); - if (!_useBitVector || (_bitVector.get() == NULL)) { + if (!_useBitVector || !_bitVector) { _postingHandle = _diskIndex.readPostingList(*_lookupRes); } _fetchPostingsDone = true; @@ -80,7 +80,7 @@ DiskTermBlueprint::fetchPostings(bool strict) SearchIterator::UP DiskTermBlueprint::createLeafSearch(const TermFieldMatchDataArray & tfmda, bool strict) const { - if ((_bitVector.get() != NULL) && (_useBitVector || (tfmda[0]->isNotNeeded() && !_hasEquivParent))) { + if (_bitVector && (_useBitVector || (tfmda[0]->isNotNeeded() && !_hasEquivParent))) { LOG(debug, "Return BitVectorIterator: %s, wordNum(%" PRIu64 "), docCount(%" PRIu64 ")", getName(_lookupRes->indexId).c_str(), _lookupRes->wordNum, _lookupRes->counts._numDocs); return BitVectorIterator::create(_bitVector.get(), tfmda, strict); diff --git a/searchlib/src/vespa/searchlib/diskindex/docidmapper.cpp b/searchlib/src/vespa/searchlib/diskindex/docidmapper.cpp index 70f7bdfea10..602b8143fd1 100644 --- a/searchlib/src/vespa/searchlib/diskindex/docidmapper.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/docidmapper.cpp @@ -11,7 +11,7 @@ namespace search::diskindex { DocIdMapping::DocIdMapping() : _docIdLimit(0u), - _selector(NULL), + _selector(nullptr), _selectorId(0) { } @@ -21,7 +21,7 @@ void DocIdMapping::clear() { _docIdLimit = 0; - _selector = NULL; + _selector = nullptr; _selectorId = 0; } @@ -30,7 +30,7 @@ void DocIdMapping::setup(uint32_t docIdLimit) { _docIdLimit = docIdLimit; - _selector = NULL; + _selector = nullptr; _selectorId = 0; } @@ -50,8 +50,7 @@ bool DocIdMapping::readDocIdLimit(const vespalib::string &mergedDir) { uint32_t docIdLimit = 0; - if (!search::docsummary::DocumentSummary:: - readDocIdLimit(mergedDir, docIdLimit)) + if (!search::docsummary::DocumentSummary::readDocIdLimit(mergedDir, docIdLimit)) return false; _docIdLimit = docIdLimit; return true; diff --git a/searchlib/src/vespa/searchlib/diskindex/docidmapper.h b/searchlib/src/vespa/searchlib/diskindex/docidmapper.h index 415405f8ae8..169e06a64ca 100644 --- a/searchlib/src/vespa/searchlib/diskindex/docidmapper.h +++ b/searchlib/src/vespa/searchlib/diskindex/docidmapper.h @@ -5,11 +5,9 @@ #include <vespa/vespalib/stllike/string.h> #include <cassert> -namespace search { +namespace search { class BitVector; } -class BitVector; - -namespace diskindex { +namespace search::diskindex { typedef vespalib::Array<uint8_t> SelectorArray; @@ -37,17 +35,17 @@ public: uint8_t _selectorId; DocIdMapper() - : _selector(NULL), + : _selector(nullptr), _docIdLimit(0u), _selectorLimit(0), _selectorId(0u) { } void setup(const DocIdMapping &mapping) { - _selector = (mapping._selector != NULL) ? - &((*mapping._selector)[0]) : NULL; + _selector = (mapping._selector != nullptr) ? + &((*mapping._selector)[0]) : nullptr; _docIdLimit = mapping._docIdLimit; - _selectorLimit = (mapping._selector != NULL) ? + _selectorLimit = (mapping._selector != nullptr) ? (*mapping._selector).size() : 0u; _selectorId = mapping._selectorId; @@ -59,7 +57,7 @@ public: uint32_t mapDocId(uint32_t docId) const { assert(docId < _docIdLimit); - if (_selector != NULL && + if (_selector != nullptr && (docId >= _selectorLimit || _selector[docId] != _selectorId)) { docId = noDocId(); } @@ -67,7 +65,4 @@ public: } }; -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp b/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp index c6f9cc757fd..7b9a9b4bdd4 100644 --- a/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/extposocc.cpp @@ -22,18 +22,13 @@ using search::index::PostingListCounts; using search::index::PostingListParams; using search::index::Schema; -namespace -{ +namespace { vespalib::string PosOccIdCooked = "PosOcc.1.Cooked"; } -namespace search -{ - -namespace diskindex -{ +namespace search::diskindex { void setupDefaultPosOccParameters(PostingListParams *countParams, @@ -66,7 +61,7 @@ makePosOccWrite(const vespalib::string &name, uint32_t indexId, const TuneFileSeqWrite &tuneFileWrite) { - PostingListFileSeqWrite *posOccWrite = NULL; + PostingListFileSeqWrite *posOccWrite = nullptr; FileHeader fileHeader; if (fileHeader.taste(name, tuneFileWrite)) { @@ -111,7 +106,7 @@ makePosOccRead(const vespalib::string &name, const PostingListParams &featureParams, const TuneFileSeqRead &tuneFileRead) { - PostingListFileSeqRead *posOccRead = NULL; + PostingListFileSeqRead *posOccRead = nullptr; FileHeader fileHeader; if (fileHeader.taste(name, tuneFileRead)) { @@ -146,7 +141,4 @@ makePosOccRead(const vespalib::string &name, return posOccRead; } - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/extposocc.h b/searchlib/src/vespa/searchlib/diskindex/extposocc.h index 12c57402def..df4e98abe35 100644 --- a/searchlib/src/vespa/searchlib/diskindex/extposocc.h +++ b/searchlib/src/vespa/searchlib/diskindex/extposocc.h @@ -4,25 +4,21 @@ #include <vespa/vespalib/stllike/string.h> -namespace search -{ - -class TuneFileSeqRead; -class TuneFileSeqWrite; - -namespace index { - -class PostingListParams; -class PostingListCountFileSeqWrite; -class PostingListCountFileSeqRead; -class PostingListFileSeqWrite; -class PostingListFileSeqRead; -class Schema; +namespace search { + class TuneFileSeqRead; + class TuneFileSeqWrite; +} +namespace search::index { + class PostingListParams; + class PostingListCountFileSeqWrite; + class PostingListCountFileSeqRead; + class PostingListFileSeqWrite; + class PostingListFileSeqRead; + class Schema; } -namespace diskindex -{ +namespace search::diskindex { void @@ -48,7 +44,4 @@ makePosOccRead(const vespalib::string &name, const index::PostingListParams &featureParams, const TuneFileSeqRead &tuneFileRead); -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp index cffc2e09ef8..a56771f179a 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.cpp @@ -40,9 +40,7 @@ FieldReader::FieldReader() } -FieldReader::~FieldReader() -{ -} +FieldReader::~FieldReader() = default; void diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h index d55aa39d491..a73ffa149a9 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldreader.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldreader.h @@ -11,11 +11,7 @@ #include "docidmapper.h" #include "fieldwriter.h" -namespace search -{ - -namespace diskindex -{ +namespace search::diskindex { class FieldReaderFieldInfo; @@ -58,83 +54,48 @@ protected: uint32_t _docIdLimit; vespalib::string _word; - static uint64_t - noWordNumHigh() - { + static uint64_t noWordNumHigh() { return std::numeric_limits<uint64_t>::max(); } - static uint64_t - noWordNum() - { + static uint64_t noWordNum() { return 0u; } - void - readCounts(); - - void - readDocIdAndFeatures(); + void readCounts(); + void readDocIdAndFeatures(); public: FieldReader(); - virtual - ~FieldReader(); - - virtual void - read(); + virtual ~FieldReader(); - virtual bool - allowRawFeatures(); + virtual void read(); + virtual bool allowRawFeatures(); - void - write(FieldWriter &writer) - { + void write(FieldWriter &writer) { if (_wordNum != writer.getSparseWordNum()) { writer.newWord(_wordNum, _word); } writer.add(_docIdAndFeatures); } - bool - isValid() const - { - return _wordNum != noWordNumHigh(); - } + bool isValid() const { return _wordNum != noWordNumHigh(); } - bool - operator<(const FieldReader &rhs) const - { + bool operator<(const FieldReader &rhs) const { return _wordNum < rhs._wordNum || (_wordNum == rhs._wordNum && _docIdAndFeatures._docId < rhs._docIdAndFeatures._docId); } - virtual void - setup(const WordNumMapping &wordNumMapping, - const DocIdMapping &docIdMapping); - - virtual bool - open(const vespalib::string &prefix, const TuneFileSeqRead &tuneFileRead); - - virtual bool - close(); - - virtual void - setFeatureParams(const PostingListParams ¶ms); - - virtual void - getFeatureParams(PostingListParams ¶ms); + virtual void setup(const WordNumMapping &wordNumMapping, const DocIdMapping &docIdMapping); + virtual bool open(const vespalib::string &prefix, const TuneFileSeqRead &tuneFileRead); + virtual bool close(); + virtual void setFeatureParams(const PostingListParams ¶ms); + virtual void getFeatureParams(PostingListParams ¶ms); + uint32_t getDocIdLimit() const { return _docIdLimit; } - uint32_t - getDocIdLimit() const - { - return _docIdLimit; - } - - static std::unique_ptr<FieldReader> - allocFieldReader(const IndexIterator &index, const Schema &oldSchema); + static std::unique_ptr<FieldReader> allocFieldReader(const IndexIterator &index, const Schema &oldSchema); }; @@ -149,13 +110,8 @@ private: public: FieldReaderEmpty(const IndexIterator &index); - - virtual bool - open(const vespalib::string &prefix, const TuneFileSeqRead &tuneFileRead) - override; - - virtual void - getFeatureParams(PostingListParams ¶ms) override; + bool open(const vespalib::string &prefix, const TuneFileSeqRead &tuneFileRead) override; + void getFeatureParams(PostingListParams ¶ms) override; }; /* @@ -169,18 +125,9 @@ private: bool _hasElementWeights; public: FieldReaderStripInfo(const IndexIterator &index); - - virtual bool - allowRawFeatures() override; - - virtual void - read() override; - - virtual void - getFeatureParams(PostingListParams ¶ms) override; + bool allowRawFeatures() override; + void read() override; + void getFeatureParams(PostingListParams ¶ms) override; }; - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp index ab90a1dacc5..abc7d573db3 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.cpp @@ -30,7 +30,7 @@ FieldWriter::FieldWriter(uint32_t docIdLimit, { } -FieldWriter::~FieldWriter() { } +FieldWriter::~FieldWriter() = default; bool FieldWriter::open(const vespalib::string &prefix, @@ -170,14 +170,12 @@ FieldWriter::close() return ret; } - void FieldWriter::setFeatureParams(const PostingListParams ¶ms) { _posoccfile->setFeatureParams(params); } - void FieldWriter::getFeatureParams(PostingListParams ¶ms) { @@ -197,14 +195,14 @@ static const char *termOccNames[] = "dictionary.spdat", "dictionary.ssdat", "dictionary.words", - NULL, + nullptr, }; void FieldWriter::remove(const vespalib::string &prefix) { - for (const char **j = termOccNames; *j != NULL; ++j) { + for (const char **j = termOccNames; *j != nullptr; ++j) { vespalib::string tmpName = prefix + *j; FastOS_File::Delete(tmpName.c_str()); } diff --git a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h index 74f515958fe..9a6edf90243 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h +++ b/searchlib/src/vespa/searchlib/diskindex/fieldwriter.h @@ -8,9 +8,7 @@ #include <vespa/searchlib/bitcompression/countcompression.h> #include <vespa/searchlib/bitcompression/posocccompression.h> -namespace search { - -namespace diskindex { +namespace search::diskindex { /* * FieldWriter is used to write a dictionary and posting list file @@ -22,11 +20,6 @@ namespace diskindex { class FieldWriter { private: - FieldWriter(const FieldWriter &rhs) = delete; - FieldWriter(const FieldWriter &&rhs) = delete; - FieldWriter &operator=(const FieldWriter &rhs) = delete; - FieldWriter &operator=(const FieldWriter &&rhs) = delete; - uint64_t _wordNum; uint32_t _prevDocId; @@ -55,6 +48,10 @@ private: void flush(); public: + FieldWriter(const FieldWriter &rhs) = delete; + FieldWriter(const FieldWriter &&rhs) = delete; + FieldWriter &operator=(const FieldWriter &rhs) = delete; + FieldWriter &operator=(const FieldWriter &&rhs) = delete; FieldWriter(uint32_t docIdLimit, uint64_t numWordIds); ~FieldWriter(); @@ -83,7 +80,4 @@ public: static void remove(const vespalib::string &prefix); }; -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp index b35c25fcfd3..5861f50d738 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp @@ -16,9 +16,7 @@ using bitcompression::FeatureDecodeContextBE; FileHeader::FileHeader() : _bigEndian(false), - _hostEndian(false), _completed(false), - _allowNoFileBitSize(false), _version(0), _headerLen(0), _fileBitSize(0), @@ -26,7 +24,7 @@ FileHeader::FileHeader() { } -FileHeader::~FileHeader() {} +FileHeader::~FileHeader() = default; bool FileHeader::taste(const vespalib::string &name, @@ -73,7 +71,7 @@ FileHeader::taste(const vespalib::string &name, return false; } } - _hostEndian = _bigEndian == (htonl(1) == 1); + if (header.hasTag("frozen")) { _completed = header.getTag("frozen").asInteger() != 0; } else { @@ -92,7 +90,7 @@ FileHeader::taste(const vespalib::string &name, name.c_str(), _fileBitSize, fileSize); LOG_ABORT("should not be reached"); } - } else if (!_allowNoFileBitSize) { + } else { LOG(error, "FileHeader::taste(\"%s\"): Missing fileBitSize tag", name.c_str()); return false; } diff --git a/searchlib/src/vespa/searchlib/diskindex/fileheader.h b/searchlib/src/vespa/searchlib/diskindex/fileheader.h index 3d8cd5f1d1b..2ddf6b9c85b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fileheader.h +++ b/searchlib/src/vespa/searchlib/diskindex/fileheader.h @@ -6,19 +6,13 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/searchlib/common/tunefileinfo.h> -namespace search -{ - -namespace diskindex -{ +namespace search::diskindex { class FileHeader { private: bool _bigEndian; - bool _hostEndian; bool _completed; - bool _allowNoFileBitSize; uint32_t _version; uint32_t _headerLen; uint64_t _fileBitSize; @@ -26,66 +20,14 @@ private: public: FileHeader(); - ~FileHeader(); - bool - taste(const vespalib::string &name, - const TuneFileSeqRead &tuneFileRead); - - bool - taste(const vespalib::string &name, - const TuneFileSeqWrite &tuneFileWrite); - - bool - taste(const vespalib::string &name, - const TuneFileRandRead &tuneFileSearch); - - bool - getBigEndian() const - { - return _bigEndian; - } - - bool - getHostEndian() const - { - return _hostEndian; - } - - uint32_t - getVersion() const - { - return _version; - } - - uint32_t - getHeaderLen() const - { - return _headerLen; - } - - const std::vector<vespalib::string> & - getFormats() const - { - return _formats; - } - - bool - getCompleted() const - { - return _completed; - } - - void - setAllowNoFileBitSize() - { - _allowNoFileBitSize = true; - } + bool taste(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead); + bool taste(const vespalib::string &name, const TuneFileSeqWrite &tuneFileWrite); + bool taste(const vespalib::string &name, const TuneFileRandRead &tuneFileSearch); + bool getBigEndian() const { return _bigEndian; } + uint32_t getVersion() const { return _version; } + const std::vector<vespalib::string> &getFormats() const { return _formats; } }; - -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp index 2db6025d257..4fa472c2584 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.cpp @@ -26,10 +26,7 @@ using search::index::SchemaUtil; using search::index::schema::DataType; using vespalib::getLastErrorString; - -namespace search { - -namespace diskindex { +namespace search::diskindex { void FusionInputIndex::setSchema(const Schema::SP &schema) @@ -40,7 +37,7 @@ FusionInputIndex::setSchema(const Schema::SP &schema) Fusion::Fusion(bool dynamicKPosIndexFormat, const TuneFileIndexing &tuneFileIndexing, const FileHeaderContext &fileHeaderContext) - : _schema(NULL), + : _schema(nullptr), _oldIndexes(), _docIdLimit(0u), _numWordIds(0u), @@ -50,27 +47,23 @@ Fusion::Fusion(bool dynamicKPosIndexFormat, _fileHeaderContext(fileHeaderContext) { } - Fusion::~Fusion() { ReleaseMappingTables(); } - void Fusion::setSchema(const Schema *schema) { _schema = schema; } - void Fusion::setOutDir(const vespalib::string &outDir) { _outDir = outDir; } - void Fusion::SetOldIndexList(const std::vector<vespalib::string> &oldIndexList) { @@ -589,7 +582,4 @@ Fusion::merge(const Schema &schema, return true; } - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/fusion.h b/searchlib/src/vespa/searchlib/diskindex/fusion.h index 76547ad0e5f..9b5806d93d9 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fusion.h +++ b/searchlib/src/vespa/searchlib/diskindex/fusion.h @@ -9,22 +9,18 @@ #include <vector> #include <string> -namespace search -{ +namespace search { template <class IN> class PostingPriorityQueue; +} -namespace common -{ - -class TuneFileIndexing; -class FileHeaderContext; - +namespace search::common { + class TuneFileIndexing; + class FileHeaderContext; } -namespace diskindex -{ +namespace search::diskindex { class FieldReader; class FieldWriter; @@ -52,68 +48,24 @@ public: { } - virtual - ~FusionInputIndex() - { - } - - void - setPath(const vespalib::string &path) - { - _path = path; - } - - const vespalib::string & - getPath() const - { - return _path; - } - - void - setTmpPath(const vespalib::string &tmpPath) - { - _tmpPath = tmpPath; - } - - const vespalib::string & - getTmpPath() const - { - return _tmpPath; - } + virtual ~FusionInputIndex() {} - const WordNumMapping & - getWordNumMapping() const - { - return _wordNumMapping; - } + void setPath(const vespalib::string &path) { _path = path; } + const vespalib::string & getPath() const { return _path; } + void setTmpPath(const vespalib::string &tmpPath) { _tmpPath = tmpPath; } + const vespalib::string &getTmpPath() const { return _tmpPath; } + const WordNumMapping & getWordNumMapping() const { return _wordNumMapping; } + WordNumMapping & getWordNumMapping() { return _wordNumMapping; } + const DocIdMapping & getDocIdMapping() const { return _docIdMapping; } - WordNumMapping & - getWordNumMapping() - { - return _wordNumMapping; - } - - const DocIdMapping & - getDocIdMapping() const - { - return _docIdMapping; - } - - DocIdMapping & - getDocIdMapping() - { - return _docIdMapping; - } + DocIdMapping & getDocIdMapping() { return _docIdMapping; } - const index::Schema & - getSchema() const - { - assert(_schema.get() != NULL); - return *_schema.get(); + const index::Schema &getSchema() const { + assert(_schema); + return *_schema; } - void - setSchema(const index::Schema::SP &schema); + void setSchema(const index::Schema::SP &schema); }; @@ -132,45 +84,28 @@ public: const TuneFileIndexing &tuneFileIndexing, const search::common::FileHeaderContext &fileHeaderContext); - virtual - ~Fusion(); + virtual ~Fusion(); void SetOldIndexList(const std::vector<vespalib::string> &oldIndexList); - bool mergeFields(); bool mergeField(uint32_t id); bool openInputFieldReaders(const SchemaUtil::IndexIterator &index, std::vector<std::unique_ptr<FieldReader> > & readers); - bool openFieldWriter(const SchemaUtil::IndexIterator &index, - FieldWriter &writer); - bool setupMergeHeap(const std::vector<std::unique_ptr<FieldReader> > & - readers, - FieldWriter &writer, - PostingPriorityQueue<FieldReader> &heap); + bool openFieldWriter(const SchemaUtil::IndexIterator &index, FieldWriter & writer); + bool setupMergeHeap(const std::vector<std::unique_ptr<FieldReader> > & readers, + FieldWriter &writer, PostingPriorityQueue<FieldReader> &heap); bool mergeFieldPostings(const SchemaUtil::IndexIterator &index); bool openInputWordReaders(const SchemaUtil::IndexIterator &index, - std::vector< - std::unique_ptr<DictionaryWordReader> > & - readers, + std::vector<std::unique_ptr<DictionaryWordReader> > &readers, PostingPriorityQueue<DictionaryWordReader> &heap); bool renumberFieldWordIds(const SchemaUtil::IndexIterator &index); - - void - setSchema(const Schema *schema); - - void - setOutDir(const vespalib::string &outDir); - + void setSchema(const Schema *schema); + void setOutDir(const vespalib::string &outDir); void makeTmpDirs(); - bool CleanTmpDirs(); - - bool - readSchemaFiles(); - - bool - checkSchemaCompat(); + bool readSchemaFiles(); + bool checkSchemaCompat(); template <class Reader, class Writer> static bool @@ -179,12 +114,6 @@ public: protected: bool ReadMappingFiles(const SchemaUtil::IndexIterator *index); bool ReleaseMappingTables(); - - static unsigned int noGen() - { - return static_cast<unsigned int>(-1); - } - protected: typedef FusionInputIndex OldIndex; @@ -210,54 +139,28 @@ protected: vespalib::string _outDir; const TuneFileIndexing &_tuneFileIndexing; - const search::common::FileHeaderContext &_fileHeaderContext; + const common::FileHeaderContext &_fileHeaderContext; - const Schema & - getSchema() const - { - assert(_schema != NULL); + const Schema &getSchema() const { + assert(_schema != nullptr); return *_schema; } public: - void - setDocIdLimit(uint32_t docIdLimit) - { - _docIdLimit = docIdLimit; - } - - void - setNumWordIds(uint64_t numWordIds) - { - _numWordIds = numWordIds; - } - - std::vector<std::shared_ptr<OldIndex> > & - getOldIndexes() - { - return _oldIndexes; - } - - virtual OldIndex * - allocOldIndex() - { - return new OldIndex; - } + void setDocIdLimit(uint32_t docIdLimit) { _docIdLimit = docIdLimit; } + std::vector<std::shared_ptr<OldIndex> > & getOldIndexes() { return _oldIndexes; } + virtual OldIndex *allocOldIndex() { return new OldIndex; } /** * This method is used by new indexing pipeline to merge indexes. */ - static bool - merge(const Schema &schema, - const vespalib::string &dir, - const std::vector<vespalib::string> &sources, - const SelectorArray &docIdSelector, - bool dynamicKPosOccFormat, - const TuneFileIndexing &tuneFileIndexing, - const search::common::FileHeaderContext &fileHeaderContext); + static bool merge(const Schema &schema, + const vespalib::string &dir, + const std::vector<vespalib::string> &sources, + const SelectorArray &docIdSelector, + bool dynamicKPosOccFormat, + const TuneFileIndexing &tuneFileIndexing, + const common::FileHeaderContext &fileHeaderContext); }; -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp index f14d158d616..91b7a813755 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.cpp @@ -13,9 +13,7 @@ LOG_SETUP(".diskindex.indexbuilder"); -namespace search { - -namespace diskindex { +namespace search::diskindex { namespace { @@ -28,7 +26,7 @@ using index::WordDocElementFeatures; using index::schema::DataType; using vespalib::getLastErrorString; -static uint32_t +uint32_t noWordPos() { return std::numeric_limits<uint32_t>::max(); @@ -59,13 +57,6 @@ public: } -inline IndexBuilder::FieldHandle & -IndexBuilder::getIndexFieldHandle(uint32_t fieldId) -{ - return _fields[fieldId]; -} - - class IndexBuilder::FieldHandle { public: @@ -269,7 +260,7 @@ public: FileHandle::FileHandle() - : _fieldWriter(NULL), + : _fieldWriter(nullptr), _docIdAndFeatures() { } @@ -288,7 +279,7 @@ FileHandle::open(vespalib::stringref dir, const TuneFileSeqWrite &tuneFileWrite, const FileHeaderContext &fileHeaderContext) { - assert(_fieldWriter == NULL); + assert(_fieldWriter == nullptr); _fieldWriter = new FieldWriter(docIdLimit, numWordIds); @@ -306,10 +297,10 @@ void FileHandle::close() { bool ret = true; - if (_fieldWriter != NULL) { + if (_fieldWriter != nullptr) { bool closeRes = _fieldWriter->close(); delete _fieldWriter; - _fieldWriter = NULL; + _fieldWriter = nullptr; if (!closeRes) { LOG(error, "Could not close term writer"); @@ -340,9 +331,7 @@ IndexBuilder::FieldHandle::FieldHandle(const Schema &schema, } -IndexBuilder::FieldHandle::~FieldHandle() -{ -} +IndexBuilder::FieldHandle::~FieldHandle() = default; void @@ -523,7 +512,7 @@ SingleIterator::appendFeatures(DocIdAndFeatures &features) IndexBuilder::IndexBuilder(const Schema &schema) : index::IndexBuilder(schema), - _currentField(NULL), + _currentField(nullptr), _curDocId(noDocId()), _lowestOKDocId(1u), _curWord(), @@ -546,11 +535,7 @@ IndexBuilder::IndexBuilder(const Schema &schema) } } - -IndexBuilder::~IndexBuilder() -{ -} - +IndexBuilder::~IndexBuilder() = default; void IndexBuilder::startWord(vespalib::stringref word) @@ -568,7 +553,7 @@ void IndexBuilder::endWord() { assert(_inWord); - assert(_currentField != NULL); + assert(_currentField != nullptr); _currentField->endWord(); _inWord = false; _lowestOKDocId = 1u; @@ -581,7 +566,7 @@ IndexBuilder::startDocument(uint32_t docId) assert(_curDocId == noDocId()); assert(docId >= _lowestOKDocId); assert(docId < _docIdLimit); - assert(_currentField != NULL); + assert(_currentField != nullptr); _curDocId = docId; assert(_curDocId != noDocId()); _currentField->startDocument(docId); @@ -592,7 +577,7 @@ void IndexBuilder::endDocument() { assert(_curDocId != noDocId()); - assert(_currentField != NULL); + assert(_currentField != nullptr); _currentField->endDocument(); _lowestOKDocId = _curDocId + 1; _curDocId = noDocId(); @@ -603,11 +588,11 @@ void IndexBuilder::startField(uint32_t fieldId) { assert(_curDocId == noDocId()); - assert(_currentField == NULL); + assert(_currentField == nullptr); assert(fieldId < _fields.size()); assert(fieldId >= _lowestOKFieldId); _currentField = &_fields[fieldId]; - assert(_currentField != NULL); + assert(_currentField != nullptr); } @@ -616,9 +601,9 @@ IndexBuilder::endField() { assert(_curDocId == noDocId()); assert(!_inWord); - assert(_currentField != NULL); + assert(_currentField != nullptr); _lowestOKFieldId = _currentField->_fieldId + 1; - _currentField = NULL; + _currentField = nullptr; } @@ -627,7 +612,7 @@ IndexBuilder::startElement(uint32_t elementId, int32_t weight, uint32_t elementLen) { - assert(_currentField != NULL); + assert(_currentField != nullptr); _currentField->startElement(elementId, weight, elementLen); } @@ -635,7 +620,7 @@ IndexBuilder::startElement(uint32_t elementId, void IndexBuilder::endElement() { - assert(_currentField != NULL); + assert(_currentField != nullptr); _currentField->endElement(); } @@ -643,7 +628,7 @@ IndexBuilder::endElement() void IndexBuilder::addOcc(const WordDocElementWordPosFeatures &features) { - assert(_currentField != NULL); + assert(_currentField != nullptr); _currentField->addOcc(features); } @@ -710,7 +695,4 @@ IndexBuilder::close() } } - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.h b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.h index dd5baf7c699..fa818bf08e6 100644 --- a/searchlib/src/vespa/searchlib/diskindex/indexbuilder.h +++ b/searchlib/src/vespa/searchlib/diskindex/indexbuilder.h @@ -7,11 +7,9 @@ #include <limits> #include <vector> -namespace search { +namespace search::common { class FileHeaderContext; } -namespace common { class FileHeaderContext; } - -namespace diskindex { +namespace search::diskindex { class BitVectorCandidate; @@ -49,37 +47,30 @@ public: // schema argument must live until indexbuilder has been deleted. IndexBuilder(const Schema &schema); - virtual ~IndexBuilder(); - - virtual void startWord(vespalib::stringref word) override; - virtual void endWord() override; - virtual void startDocument(uint32_t docId) override; - virtual void endDocument() override; - virtual void startField(uint32_t fieldId) override; - virtual void endField() override; - virtual void startElement(uint32_t elementId, int32_t weight, uint32_t elementLen) override; - virtual void endElement() override; - virtual void addOcc(const WordDocElementWordPosFeatures &features) override; + ~IndexBuilder() override; + + void startWord(vespalib::stringref word) override; + void endWord() override; + void startDocument(uint32_t docId) override; + void endDocument() override; + void startField(uint32_t fieldId) override; + void endField() override; + void startElement(uint32_t elementId, int32_t weight, uint32_t elementLen) override; + void endElement() override; + void addOcc(const WordDocElementWordPosFeatures &features) override; // TODO: methods for attribute vectors. // TODO: methods for document summary. - inline FieldHandle & getIndexFieldHandle(uint32_t fieldId); void setPrefix(vespalib::stringref prefix); vespalib::string appendToPrefix(vespalib::stringref name); - void - open(uint32_t docIdLimit, uint64_t numWordIds, - const TuneFileIndexing &tuneFileIndexing, - const search::common::FileHeaderContext &fileHandleContext); + void open(uint32_t docIdLimit, uint64_t numWordIds, + const TuneFileIndexing &tuneFileIndexing, + const common::FileHeaderContext &fileHandleContext); void close(); }; -} // namespace diskindex - -} // namespace search - - - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp index 1870de30bad..7621b178634 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.cpp @@ -42,8 +42,8 @@ const uint32_t headerAlign = 4096; } PageDict4FileSeqRead::PageDict4FileSeqRead() - : _pReader(NULL), - _ssReader(NULL), + : _pReader(nullptr), + _ssReader(nullptr), _ssd(), _ssReadContext(_ssd), _ssfile(), @@ -261,15 +261,15 @@ PageDict4FileSeqRead::close() { delete _pReader; delete _ssReader; - _pReader = NULL; - _ssReader = NULL; + _pReader = nullptr; + _ssReader = nullptr; _ssReadContext.dropComprBuf(); _spReadContext.dropComprBuf(); _pReadContext.dropComprBuf(); - _ssReadContext.setFile(NULL); - _spReadContext.setFile(NULL); - _pReadContext.setFile(NULL); + _ssReadContext.setFile(nullptr); + _spReadContext.setFile(nullptr); + _pReadContext.setFile(nullptr); _ssfile.Close(); _spfile.Close(); _pfile.Close(); @@ -290,9 +290,9 @@ PageDict4FileSeqRead::getParams(PostingListParams ¶ms) PageDict4FileSeqWrite::PageDict4FileSeqWrite() - : _pWriter(NULL), - _spWriter(NULL), - _ssWriter(NULL), + : _pWriter(), + _spWriter(), + _ssWriter(), _pe(), _pWriteContext(_pe), _pfile(), @@ -312,17 +312,11 @@ PageDict4FileSeqWrite::PageDict4FileSeqWrite() } -PageDict4FileSeqWrite::~PageDict4FileSeqWrite() -{ - delete _pWriter; - delete _spWriter; - delete _ssWriter; -} +PageDict4FileSeqWrite::~PageDict4FileSeqWrite() = default; void -PageDict4FileSeqWrite::writeWord(vespalib::stringref word, - const PostingListCounts &counts) +PageDict4FileSeqWrite::writeWord(vespalib::stringref word, const PostingListCounts &counts) { _pWriter->addCounts(word, counts); } @@ -333,9 +327,9 @@ PageDict4FileSeqWrite::open(const vespalib::string &name, const TuneFileSeqWrite &tuneFileWrite, const FileHeaderContext &fileHeaderContext) { - assert(_pWriter == NULL); - assert(_spWriter == NULL); - assert(_ssWriter == NULL); + assert( ! _pWriter); + assert( ! _spWriter); + assert( ! _ssWriter); vespalib::string pname = name + ".pdat"; vespalib::string spname = name + ".spdat"; @@ -401,9 +395,9 @@ PageDict4FileSeqWrite::open(const vespalib::string &name, makeSPHeader(fileHeaderContext); makeSSHeader(fileHeaderContext); - _ssWriter = new SSWriter(_sse); - _spWriter = new SPWriter(*_ssWriter, _spe); - _pWriter = new PWriter(*_spWriter, _pe); + _ssWriter = std::make_unique<SSWriter>(_sse); + _spWriter = std::make_unique<SPWriter>(*_ssWriter, _spe); + _pWriter = std::make_unique<PWriter>(*_spWriter, _pe); _spWriter->setup(); _pWriter->setup(); @@ -428,27 +422,24 @@ PageDict4FileSeqWrite::close() _pWriteContext.dropComprBuf(); _pfile.Sync(); _pfile.Close(); - _pWriteContext.setFile(NULL); + _pWriteContext.setFile(nullptr); _spWriteContext.dropComprBuf(); _spfile.Sync(); _spfile.Close(); - _spWriteContext.setFile(NULL); + _spWriteContext.setFile(nullptr); _ssWriteContext.dropComprBuf(); _ssfile.Sync(); _ssfile.Close(); - _ssWriteContext.setFile(NULL); + _ssWriteContext.setFile(nullptr); // Update file headers updatePHeader(usedPBits); updateSPHeader(usedSPBits); updateSSHeader(usedSSBits); - delete _pWriter; - delete _spWriter; - delete _ssWriter; - _pWriter = NULL; - _spWriter = NULL; - _ssWriter = NULL; + _pWriter.reset(); + _spWriter.reset(); + _ssWriter.reset(); return true; } diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h index e176ba26d88..2363c89ac3b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4file.h @@ -50,44 +50,22 @@ class PageDict4FileSeqRead : public index::DictionaryFileSeqRead uint64_t _wordNum; - void - readSSHeader(); - - void - readSPHeader(); - - void - readPHeader(); - + void readSSHeader(); + void readSPHeader(); + void readPHeader(); public: PageDict4FileSeqRead(); - - virtual - ~PageDict4FileSeqRead(); + ~PageDict4FileSeqRead() override; /** * Read word and counts. Only nonzero counts are returned. If at * end of dictionary then noWordNumHigh() is returned as word number. */ - virtual void - readWord(vespalib::string &word, - uint64_t &wordNum, - PostingListCounts &counts) override; - - virtual bool open(const vespalib::string &name, - const TuneFileSeqRead &tuneFileRead) override; - - /** - * Close dictionary file. - */ - virtual bool close() override; - - /* - * Get current parameters. - */ - virtual void - getParams(index::PostingListParams ¶ms) override; + void readWord(vespalib::string &word, uint64_t &wordNum, PostingListCounts &counts) override; + bool open(const vespalib::string &name, const TuneFileSeqRead &tuneFileRead) override; + bool close() override; + void getParams(index::PostingListParams ¶ms) override; }; /** @@ -104,10 +82,11 @@ class PageDict4FileSeqWrite : public index::DictionaryFileSeqWrite typedef bitcompression::PageDict4PWriter PWriter; typedef index::PostingListCounts PostingListCounts; + using FileHeaderContext = common::FileHeaderContext; - PWriter *_pWriter; - SPWriter *_spWriter; - SSWriter *_ssWriter; + std::unique_ptr<PWriter> _pWriter; + std::unique_ptr<SPWriter> _spWriter; + std::unique_ptr<SSWriter> _ssWriter; EC _pe; ComprFileWriteContext _pWriteContext; @@ -125,69 +104,29 @@ class PageDict4FileSeqWrite : public index::DictionaryFileSeqWrite uint32_t _spHeaderLen; // Length of header for sparse page file (bytes) uint32_t _ssHeaderLen; // Length of header for sparse sparse file (bytes) - void - writeIndexNames(vespalib::GenericHeader &header); - - void - writeSSSubHeader(vespalib::GenericHeader &header); - - void - makePHeader(const search::common::FileHeaderContext &fileHeaderContext); - - void - makeSPHeader(const search::common::FileHeaderContext &fileHeaderContext); - - void - makeSSHeader(const search::common::FileHeaderContext &fileHeaderContext); - - void - updatePHeader(uint64_t fileBitSize); - - void - updateSPHeader(uint64_t fileBitSize); - - void - updateSSHeader(uint64_t fileBitSize); - + void writeSSSubHeader(vespalib::GenericHeader &header); + void makePHeader(const FileHeaderContext &fileHeaderContext); + void makeSPHeader(const FileHeaderContext &fileHeaderContext); + void makeSSHeader(const FileHeaderContext &fileHeaderContext); + void updatePHeader(uint64_t fileBitSize); + void updateSPHeader(uint64_t fileBitSize); + void updateSSHeader(uint64_t fileBitSize); public: PageDict4FileSeqWrite(); - - virtual ~PageDict4FileSeqWrite(); - /** - * Write word and counts. Only nonzero counts should be supplied. - */ - virtual void - writeWord(vespalib::stringref word, - const PostingListCounts &counts) override; + void writeWord(vespalib::stringref word, const PostingListCounts &counts) override; /** * Open dictionary file for sequential write. The index with most * words should be first for optimal compression. */ - virtual bool - open(const vespalib::string &name, - const TuneFileSeqWrite &tuneFileWrite, - const search::common::FileHeaderContext &fileHeaderContext) override; + bool open(const vespalib::string &name, const TuneFileSeqWrite &tuneFileWrite, + const FileHeaderContext &fileHeaderContext) override; - /** - * Close dictionary file. - */ - virtual bool - close() override; - - /* - * Set parameters. - */ - virtual void - setParams(const index::PostingListParams ¶ms) override; - - /* - * Get current parameters. - */ - virtual void - getParams(index::PostingListParams ¶ms) override; + bool close() override; + void setParams(const index::PostingListParams ¶ms) override; + void getParams(index::PostingListParams ¶ms) override; }; } diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp index 67f955fab07..4ee37470e1d 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.cpp @@ -18,19 +18,16 @@ vespalib::string mySSId("PageDict4SS.1"); using vespalib::getLastErrorString; -namespace search { - -namespace diskindex { - +namespace search::diskindex { PageDict4RandRead::PageDict4RandRead() : DictionaryFileRandRead(), _ssReader(), _ssd(), _ssReadContext(_ssd), - _ssfile(new FastOS_File()), - _spfile(new FastOS_File()), - _pfile(new FastOS_File()), + _ssfile(std::make_unique<FastOS_File>()), + _spfile(std::make_unique<FastOS_File>()), + _pfile(std::make_unique<FastOS_File>()), _ssFileBitSize(0u), _spFileBitSize(0u), _pFileBitSize(0u), @@ -42,7 +39,7 @@ PageDict4RandRead::PageDict4RandRead() } -PageDict4RandRead::~PageDict4RandRead() { } +PageDict4RandRead::~PageDict4RandRead() = default; void @@ -179,8 +176,7 @@ PageDict4RandRead::lookup(vespalib::stringref word, ssRes._pageNum); PLookupRes pRes; - const char *pData = static_cast<const char *> - (_pfile->MemoryMapPtr(0)); + const char *pData = static_cast<const char *>(_pfile->MemoryMapPtr(0)); pRes.lookup(*_ssReader, pData + pageSize * spRes._pageNum, word, @@ -209,16 +205,15 @@ PageDict4RandRead::open(const vespalib::string &name, vespalib::string spname = name + ".spdat"; vespalib::string ssname = name + ".ssdat"; - if (tuneFileRead.getWantMemoryMap() || true) { - int mmapFlags(tuneFileRead.getMemoryMapFlags()); - _ssfile->enableMemoryMap(mmapFlags); - _spfile->enableMemoryMap(mmapFlags); - _pfile->enableMemoryMap(mmapFlags); - } else if (tuneFileRead.getWantDirectIO()) { - _ssfile->EnableDirectIO(); - _spfile->EnableDirectIO(); - _pfile->EnableDirectIO(); - } + int mmapFlags(tuneFileRead.getMemoryMapFlags()); + _ssfile->enableMemoryMap(mmapFlags); + _spfile->enableMemoryMap(mmapFlags); + _pfile->enableMemoryMap(mmapFlags); + + int fadvise = tuneFileRead.getAdvise(); + _ssfile->setFAdviseOptions(fadvise); + _spfile->setFAdviseOptions(fadvise); + _pfile->setFAdviseOptions(fadvise); if (!_ssfile->OpenReadOnly(ssname.c_str())) { LOG(error, "could not open %s: %s", _ssfile->GetFileName(), getLastErrorString().c_str()); @@ -259,21 +254,17 @@ PageDict4RandRead::close() _ssReader.reset(); _ssReadContext.dropComprBuf(); - _ssReadContext.setFile(NULL); + _ssReadContext.setFile(nullptr); _ssfile->Close(); _spfile->Close(); _pfile->Close(); return true; } - uint64_t PageDict4RandRead::getNumWordIds() const { return _ssd._numWordIds; } - -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.h b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.h index b7159f9398e..4da77880692 100644 --- a/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.h +++ b/searchlib/src/vespa/searchlib/diskindex/pagedict4randread.h @@ -7,9 +7,7 @@ #include <vespa/searchlib/bitcompression/countcompression.h> #include <vespa/searchlib/bitcompression/pagedict4.h> -namespace search { - -namespace diskindex { +namespace search::diskindex { class PageDict4RandRead : public index::DictionaryFileRandRead { @@ -55,6 +53,4 @@ public: uint64_t getNumWordIds() const override; }; -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp index 3bd879de291..bc911e230bc 100644 --- a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.cpp @@ -64,37 +64,4 @@ WordNumMapping::setup(uint32_t numWordIds) _oldDictSize = numWordIds; } - -void -WordNumMapper::sanityCheck(bool allowHoles) -{ - uint64_t dictSize = getMaxWordNum(); - uint64_t mappedWordNum = map(0u); - assert(mappedWordNum == 0u); - for (uint64_t wordNum = 1; wordNum <= dictSize; ++wordNum) { - uint64_t prevMappedWordNum = mappedWordNum; - mappedWordNum = map(wordNum); - if (mappedWordNum == 0u && allowHoles) - continue; // In case some words are being removed - assert(mappedWordNum > prevMappedWordNum); - (void) prevMappedWordNum; - } -} - - -uint64_t -WordNumMapping::getMaxMappedWordNum() const -{ - WordNumMapper mapper(*this); - return mapper.getMaxMappedWordNum(); -} - - -void -WordNumMapping::sanityCheck(bool allowHoles) -{ - WordNumMapper mapper(*this); - mapper.sanityCheck(allowHoles); -} - } diff --git a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.h b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.h index 8141a16c137..f36d5656bd1 100644 --- a/searchlib/src/vespa/searchlib/diskindex/wordnummapper.h +++ b/searchlib/src/vespa/searchlib/diskindex/wordnummapper.h @@ -28,7 +28,7 @@ public: const uint64_t *getOld2NewWordNums() const { return (_old2newwords.empty()) - ? NULL + ? nullptr : &_old2newwords[0]; } @@ -37,8 +37,6 @@ public: void noMappingFile(); void clear(); void setup(uint32_t numWordIds); - uint64_t getMaxMappedWordNum() const; - void sanityCheck(bool allowHoles); }; @@ -55,31 +53,20 @@ class WordNumMapper public: WordNumMapper() - : _old2newwords(NULL), + : _old2newwords(nullptr), _oldDictSize(0) {} - WordNumMapper(const WordNumMapping &mapping) - : _old2newwords(NULL), - _oldDictSize(0) - { - setup(mapping); - } - void setup(const WordNumMapping &mapping) { _old2newwords = mapping.getOld2NewWordNums(); _oldDictSize = mapping.getOldDictSize(); } uint64_t map(uint32_t wordNum) const { - return (_old2newwords != NULL) + return (_old2newwords != nullptr) ? _old2newwords[wordNum] : wordNum; } - - uint64_t getMaxWordNum() const { return _oldDictSize; } - uint64_t getMaxMappedWordNum() const { return map(_oldDictSize); } - void sanityCheck(bool allowHoles); }; } diff --git a/searchlib/src/vespa/searchlib/diskindex/zcbuf.cpp b/searchlib/src/vespa/searchlib/diskindex/zcbuf.cpp index 3b7ccab0c53..e2f5e68758e 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcbuf.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcbuf.cpp @@ -7,9 +7,9 @@ namespace search::diskindex { ZcBuf::ZcBuf() - : _valI(NULL), - _valE(NULL), - _mallocStart(NULL), + : _valI(nullptr), + _valE(nullptr), + _mallocStart(nullptr), _mallocSize(0) { } diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocc.h b/searchlib/src/vespa/searchlib/diskindex/zcposocc.h index 093e452d4e1..cd21fb02f33 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocc.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocc.h @@ -5,9 +5,7 @@ #include "zcposting.h" #include <vespa/searchlib/bitcompression/posocccompression.h> -namespace search { - -namespace diskindex { +namespace search::diskindex { class Zc4PosOccSeqRead : public Zc4PostingSeqRead { @@ -59,8 +57,4 @@ public: ZcPosOccSeqWrite(const Schema &schema, uint32_t indexId, index::PostingListCountFileSeqWrite *countFile); }; - -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h index d55ba24be09..41f2b747916 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposocciterators.h @@ -5,9 +5,7 @@ #include "zcpostingiterators.h" #include <vespa/searchlib/bitcompression/posocccompression.h> -namespace search { - -namespace diskindex { +namespace search::diskindex { template <bool bigEndian> class Zc4RareWordPosOccIterator : public Zc4RareWordPostingIterator<bigEndian> @@ -21,7 +19,7 @@ private: public: Zc4RareWordPosOccIterator(Position start, uint64_t bitLength, uint32_t docIdLimit, const bitcompression::PosOccFieldsParams *fieldsParams, - const search::fef::TermFieldMatchDataArray &matchData); + const fef::TermFieldMatchDataArray &matchData); }; @@ -38,7 +36,7 @@ public: Zc4PosOccIterator(Position start, uint64_t bitLength, uint32_t docIdLimit, uint32_t minChunkDocs, const index::PostingListCounts &counts, const bitcompression::PosOccFieldsParams *fieldsParams, - const search::fef::TermFieldMatchDataArray &matchData); + const fef::TermFieldMatchDataArray &matchData); }; @@ -54,7 +52,7 @@ private: public: ZcRareWordPosOccIterator(Position start, uint64_t bitLength, uint32_t docidLimit, const bitcompression::PosOccFieldsParams *fieldsParams, - const search::fef::TermFieldMatchDataArray &matchData); + const fef::TermFieldMatchDataArray &matchData); }; @@ -71,7 +69,7 @@ public: ZcPosOccIterator(Position start, uint64_t bitLength, uint32_t docidLimit, uint32_t minChunkDocs, const index::PostingListCounts &counts, const bitcompression::PosOccFieldsParams *fieldsParams, - const search::fef::TermFieldMatchDataArray &matchData); + const fef::TermFieldMatchDataArray &matchData); }; @@ -87,7 +85,4 @@ extern template class ZcRareWordPosOccIterator<false>; extern template class ZcPosOccIterator<true>; extern template class ZcPosOccIterator<false>; -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp index c05001c2ab8..b8d5ee89276 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.cpp @@ -28,14 +28,12 @@ vespalib::string myId5("Zc.5"); } -namespace search { - -namespace diskindex { +namespace search::diskindex { using vespalib::getLastErrorString; ZcPosOccRandRead::ZcPosOccRandRead() - : _file(new FastOS_File()), + : _file(std::make_unique<FastOS_File>()), _fileSize(0), _minChunkDocs(1 << 30), _minSkipDocs(64), @@ -62,9 +60,6 @@ createIterator(const PostingListCounts &counts, const search::fef::TermFieldMatchDataArray &matchData, bool usebitVector) const { - (void) counts; - (void) handle; - (void) matchData; (void) usebitVector; typedef EGPosOccEncodeContext<true> EC; @@ -125,9 +120,9 @@ ZcPosOccRandRead::readPostingList(const PostingListCounts &counts, startOffset -= (startOffset & 7); void *mapPtr = _file->MemoryMapPtr(startOffset); - if (mapPtr != NULL) { + if (mapPtr != nullptr) { handle._mem = mapPtr; - handle._allocMem = NULL; + handle._allocMem = nullptr; handle._allocSize = 0; } else { uint64_t endOffset = (handle._bitOffset + _headerBitSize + @@ -145,11 +140,11 @@ ZcPosOccRandRead::readPostingList(const PostingListCounts &counts, padExtraAfter = 16 - padAfter; size_t mallocLen = padBefore + vectorLen + padAfter + padExtraAfter; - void *mallocStart = NULL; - void *alignedBuffer = NULL; + void *mallocStart = nullptr; + void *alignedBuffer = nullptr; if (mallocLen > 0) { alignedBuffer = _file->AllocateDirectIOBuffer(mallocLen, mallocStart); - assert(mallocStart != NULL); + assert(mallocStart != nullptr); assert(endOffset + padAfter + padExtraAfter <= _fileSize); _file->ReadBuf(alignedBuffer, padBefore + vectorLen + padAfter, @@ -173,14 +168,15 @@ bool ZcPosOccRandRead:: open(const vespalib::string &name, const TuneFileRandRead &tuneFileRead) { + _file->setFAdviseOptions(tuneFileRead.getAdvise()); if (tuneFileRead.getWantMemoryMap()) { _file->enableMemoryMap(tuneFileRead.getMemoryMapFlags()); - } else if (tuneFileRead.getWantDirectIO()) + } else if (tuneFileRead.getWantDirectIO()) { _file->EnableDirectIO(); + } bool res = _file->OpenReadOnly(name.c_str()); if (!res) { - LOG(error, "could not open %s: %s", - _file->GetFileName(), getLastErrorString().c_str()); + LOG(error, "could not open %s: %s", _file->GetFileName(), getLastErrorString().c_str()); return false; } _fileSize = _file->GetSize(); @@ -365,6 +361,4 @@ Zc4PosOccRandRead::getSubIdentifier() return d.getIdentifier(); } -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h index 0dcca879cac..3741f011c30 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcposoccrandread.h @@ -7,9 +7,7 @@ #include <vespa/searchlib/bitcompression/posocccompression.h> #include <vespa/searchlib/fef/termfieldmatchdataarray.h> -namespace search { - -namespace diskindex { +namespace search::diskindex { class ZcPosOccRandRead : public index::PostingListFileRandRead { @@ -39,20 +37,15 @@ public: * Create iterator for single word. Semantic lifetime of counts and * handle must exceed lifetime of iterator. */ - search::queryeval::SearchIterator * - createIterator(const PostingListCounts &counts, - const PostingListHandle &handle, - const search::fef::TermFieldMatchDataArray &matchData, - bool usebitVector) const override; + queryeval::SearchIterator * + createIterator(const PostingListCounts &counts, const PostingListHandle &handle, + const fef::TermFieldMatchDataArray &matchData, bool usebitVector) const override; /** * Read (possibly partial) posting list into handle. */ - void - readPostingList(const PostingListCounts &counts, - uint32_t firstSegment, - uint32_t numSegments, - PostingListHandle &handle) override; + void readPostingList(const PostingListCounts &counts, uint32_t firstSegment, + uint32_t numSegments, PostingListHandle &handle) override; bool open(const vespalib::string &name, const TuneFileRandRead &tuneFileRead) override; bool close() override; @@ -70,11 +63,9 @@ public: * Create iterator for single word. Semantic lifetime of counts and * handle must exceed lifetime of iterator. */ - search::queryeval::SearchIterator * - createIterator(const PostingListCounts &counts, - const PostingListHandle &handle, - const search::fef::TermFieldMatchDataArray &matchData, - bool usebitVector) const override; + queryeval::SearchIterator * + createIterator(const PostingListCounts &counts, const PostingListHandle &handle, + const fef::TermFieldMatchDataArray &matchData, bool usebitVector) const override; void readHeader() override; @@ -83,7 +74,4 @@ public: }; -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp index 688e3ef59e1..d51a592bf2b 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcposting.cpp @@ -78,7 +78,7 @@ Zc4PostingSeqRead(PostingListCountFileSeqRead *countFile) _wordStart(0), _residue(0) { - if (_countFile != NULL) { + if (_countFile != nullptr) { PostingListParams params; _countFile->getParams(params); params.get("docIdLimit", _docIdLimit); @@ -492,7 +492,7 @@ Zc4PostingSeqRead::close() { _readContext.dropComprBuf(); _file.Close(); - _readContext.setFile(NULL); + _readContext.setFile(nullptr); return true; } @@ -500,7 +500,7 @@ Zc4PostingSeqRead::close() void Zc4PostingSeqRead::getParams(PostingListParams ¶ms) { - if (_countFile != NULL) { + if (_countFile != nullptr) { PostingListParams countParams; _countFile->getParams(countParams); params = countParams; @@ -614,7 +614,7 @@ Zc4PostingSeqWrite(PostingListCountFileSeqWrite *countFile) _minSkipDocs(64), _docIdLimit(10000000), _docIds(), - _encodeFeatures(NULL), + _encodeFeatures(nullptr), _featureOffset(0), _featureWriteContext(sizeof(uint64_t)), _writePos(0), @@ -630,7 +630,7 @@ Zc4PostingSeqWrite(PostingListCountFileSeqWrite *countFile) { _encodeContext.setWriteContext(&_writeContext); - if (_countFile != NULL) { + if (_countFile != nullptr) { PostingListParams params; _countFile->getParams(params); params.get("docIdLimit", _docIdLimit); @@ -874,7 +874,7 @@ Zc4PostingSeqWrite::close() _writeContext.dropComprBuf(); _file.Sync(); _file.Close(); - _writeContext.setFile(NULL); + _writeContext.setFile(nullptr); updateHeader(); return true; } @@ -885,7 +885,7 @@ void Zc4PostingSeqWrite:: setParams(const PostingListParams ¶ms) { - if (_countFile != NULL) + if (_countFile != nullptr) _countFile->setParams(params); params.get("docIdLimit", _docIdLimit); params.get("minChunkDocs", _minChunkDocs); @@ -897,7 +897,7 @@ void Zc4PostingSeqWrite:: getParams(PostingListParams ¶ms) { - if (_countFile != NULL) { + if (_countFile != nullptr) { PostingListParams countParams; _countFile->getParams(countParams); params = countParams; diff --git a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp index 919004c378f..513f39c47f8 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.cpp @@ -4,9 +4,7 @@ #include <vespa/searchlib/fef/termfieldmatchdataarray.h> #include <vespa/searchlib/bitcompression/posocccompression.h> -namespace search { - -namespace diskindex { +namespace search::diskindex { using search::fef::TermFieldMatchDataArray; using search::bitcompression::FeatureDecodeContext; @@ -39,7 +37,7 @@ template <bool bigEndian> Zc4RareWordPostingIterator<bigEndian>:: Zc4RareWordPostingIterator(const TermFieldMatchDataArray &matchData, Position start, uint32_t docIdLimit) : ZcIteratorBase(matchData, start, docIdLimit), - _decodeContext(NULL), + _decodeContext(nullptr), _residue(0), _prevDocId(0), _numDocs(0) @@ -206,8 +204,8 @@ ZcRareWordPostingIterator<bigEndian>::readWordStart(uint32_t docIdLimit) ZcPostingIteratorBase::ZcPostingIteratorBase(const TermFieldMatchDataArray &matchData, Position start, uint32_t docIdLimit) : ZcIteratorBase(matchData, start, docIdLimit), - _valI(NULL), - _valIBase(NULL), + _valI(nullptr), + _valIBase(nullptr), _featureSeekPos(0), _l1(), _l2(), @@ -228,12 +226,12 @@ ZcPostingIterator(uint32_t minChunkDocs, const search::fef::TermFieldMatchDataArray &matchData, Position start, uint32_t docIdLimit) : ZcPostingIteratorBase(matchData, start, docIdLimit), - _decodeContext(NULL), + _decodeContext(nullptr), _minChunkDocs(minChunkDocs), _docIdK(0), _dynamicK(dynamicK), _numDocs(0), - _featuresValI(NULL), + _featuresValI(nullptr), _featuresBitOffset(0), _counts(counts) { } @@ -600,6 +598,4 @@ template class ZcPostingIterator<false>; template class ZcRareWordPostingIterator<true>; template class ZcRareWordPostingIterator<false>; -} // namespace diskindex - -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.h b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.h index a1f7a8b2e9f..97b7e2dc0cc 100644 --- a/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.h +++ b/searchlib/src/vespa/searchlib/diskindex/zcpostingiterators.h @@ -7,9 +7,7 @@ #include <vespa/searchlib/queryeval/iterators.h> #include <vespa/fastos/dynamiclibrary.h> -namespace search { - -namespace diskindex { +namespace search::diskindex { using bitcompression::Position; @@ -282,18 +280,15 @@ public: // Counts used for assertions const PostingListCounts &_counts; - ZcPostingIterator(uint32_t minChunkDocs, - bool dynamicK, - const PostingListCounts &counts, - const search::fef::TermFieldMatchDataArray &matchData, - Position start, uint32_t docIdLimit); + ZcPostingIterator(uint32_t minChunkDocs, bool dynamicK, const PostingListCounts &counts, + const search::fef::TermFieldMatchDataArray &matchData, Position start, uint32_t docIdLimit); void doUnpack(uint32_t docId) override; void readWordStart(uint32_t docIdLimit) override; void rewind(Position start) override; - virtual void featureSeek(uint64_t offset) override { + void featureSeek(uint64_t offset) override { _decodeContext->_valI = _featuresValI + (_featuresBitOffset + offset) / 64; _decodeContext->setupBits((_featuresBitOffset + offset) & 63); } @@ -309,8 +304,4 @@ extern template class ZcPostingIterator<false>; extern template class ZcRareWordPostingIterator<true>; extern template class ZcRareWordPostingIterator<false>; - -} // namespace diskindex - -} // namespace search - +} diff --git a/searchlib/src/vespa/searchlib/test/diskindex/testdiskindex.cpp b/searchlib/src/vespa/searchlib/test/diskindex/testdiskindex.cpp index aa6a9727352..f0bb1eb6519 100644 --- a/searchlib/src/vespa/searchlib/test/diskindex/testdiskindex.cpp +++ b/searchlib/src/vespa/searchlib/test/diskindex/testdiskindex.cpp @@ -5,15 +5,13 @@ #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/vespalib/io/fileutil.h> -namespace search { +namespace search::diskindex { using index::DummyFileHeaderContext; using index::Schema; using index::WordDocElementWordPosFeatures; using index::schema::DataType; -namespace diskindex { - struct Builder { search::diskindex::IndexBuilder _ib; @@ -106,21 +104,14 @@ TestDiskIndex::openIndex(const std::string &dir, bool directio, bool readmmap, if (readmmap) { tuneFileRead.setWantMemoryMap(); } - _index.reset(new DiskIndex(dir)); + _index = std::make_unique<DiskIndex>(dir); bool ok(_index->setup(tuneFileRead)); assert(ok); (void) ok; } -TestDiskIndex::TestDiskIndex() : - _schema(), - _index() -{ -} +TestDiskIndex::TestDiskIndex() = default; -TestDiskIndex::~TestDiskIndex() -{ -} +TestDiskIndex::~TestDiskIndex() = default; } -} diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp index f0cbefd443b..ff1d7e1d6ec 100644 --- a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp @@ -192,7 +192,7 @@ ProcessMemoryStats::create(uint64_t sizeEpsilon) if (samples.back().similarTo(*(samples.rbegin()+1), sizeEpsilon)) { return samples.back(); } - LOG(info, "create(): Memory stats have changed, trying to read smaps file again: i=%zu, prevStats={%s}, currStats={%s}", + LOG(debug, "create(): Memory stats have changed, trying to read smaps file again: i=%zu, prevStats={%s}, currStats={%s}", i, (samples.rbegin()+1)->toString().c_str(), samples.back().toString().c_str()); } std::sort(samples.begin(), samples.end()); diff --git a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp index 568bff81a1b..f5e67708fe5 100644 --- a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp @@ -1,13 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "updateoperation.h" -#include <vespa/storage/distributor/distributormetricsset.h> +#include <vespa/document/fieldvalue/document.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> -#include <vespa/document/fieldvalue/document.h> +#include <vespa/storage/distributor/distributormetricsset.h> #include <vespa/storage/distributor/distributor_bucket_space.h> +#include <vespa/vespalib/util/stringfmt.h> #include <vespa/log/log.h> + LOG_SETUP(".distributor.callback.doc.update"); @@ -20,10 +22,8 @@ UpdateOperation::UpdateOperation(DistributorComponent& manager, const std::shared_ptr<api::UpdateCommand> & msg, PersistenceOperationMetricSet& metric) : Operation(), - _trackerInstance(metric, - std::shared_ptr<api::BucketInfoReply>(new api::UpdateReply(*msg)), - manager, - msg->getTimestamp()), + _trackerInstance(metric, std::make_shared<api::UpdateReply>(*msg), + manager, msg->getTimestamp()), _tracker(_trackerInstance), _msg(msg), _manager(manager), @@ -114,23 +114,18 @@ void UpdateOperation::onReceive(DistributorMessageSender& sender, const std::shared_ptr<api::StorageReply> & msg) { - api::UpdateReply& reply = - static_cast<api::UpdateReply&>(*msg); + auto& reply = static_cast<api::UpdateReply&>(*msg); if (msg->getType() == api::MessageType::UPDATE_REPLY) { uint16_t node = _tracker.handleReply(reply); if (node != (uint16_t)-1) { if (reply.getResult().getResult() == api::ReturnCode::OK) { - _results.push_back(OldTimestamp( - reply.getBucketId(), - reply.getOldTimestamp(), - node)); + _results.emplace_back(reply.getBucketId(), reply.getOldTimestamp(), node); } if (_tracker.getReply().get()) { - api::UpdateReply& replyToSend = - static_cast<api::UpdateReply&>(*_tracker.getReply()); + auto& replyToSend = static_cast<api::UpdateReply&>(*_tracker.getReply()); uint64_t oldTs = 0; uint64_t goodNode = 0; @@ -147,12 +142,17 @@ UpdateOperation::onReceive(DistributorMessageSender& sender, for (uint32_t i = 0; i < _results.size(); i++) { if (_results[i].oldTs < oldTs) { - replyToSend.setNodeWithNewestTimestamp( - _results[goodNode].nodeId); - _newestTimestampLocation.first = - _results[goodNode].bucketId; - _newestTimestampLocation.second = - _results[goodNode].nodeId; + LOG(warning, "Update operation for '%s' in bucket %s updated documents with different timestamps. " + "This should not happen and may indicate undetected replica divergence. " + "Found ts=%zu on node %u, ts=%zu on node %u", + reply.getDocumentId().toString().c_str(), + reply.getBucket().toString().c_str(), + _results[i].oldTs, _results[i].nodeId, + _results[goodNode].oldTs, _results[goodNode].nodeId); + + replyToSend.setNodeWithNewestTimestamp(_results[goodNode].nodeId); + _newestTimestampLocation.first = _results[goodNode].bucketId; + _newestTimestampLocation.second = _results[goodNode].nodeId; break; } } diff --git a/vespabase/src/common-env.sh b/vespabase/src/common-env.sh index 018630b0622..18ae819ca96 100755 --- a/vespabase/src/common-env.sh +++ b/vespabase/src/common-env.sh @@ -123,7 +123,7 @@ export MALLOC_ARENA_MAX=1 # how to find the "java" program? # should be available in $VESPA_HOME/bin or JAVA_HOME if [ "$JAVA_HOME" ] && [ -f "${JAVA_HOME}/bin/java" ]; then - PATH="${PATH}:${JAVA_HOME}/bin" + PATH="${JAVA_HOME}/bin:${PATH}" fi VESPA_VALGRIND_SUPPREESSIONS_OPT="" |