diff options
11 files changed, 70 insertions, 44 deletions
diff --git a/container-core/src/main/java/com/yahoo/metrics/HostedNodeAdminMetrics.java b/container-core/src/main/java/com/yahoo/metrics/HostedNodeAdminMetrics.java index 5624f1f92e3..1dc58090c69 100644 --- a/container-core/src/main/java/com/yahoo/metrics/HostedNodeAdminMetrics.java +++ b/container-core/src/main/java/com/yahoo/metrics/HostedNodeAdminMetrics.java @@ -8,20 +8,20 @@ package com.yahoo.metrics; public enum HostedNodeAdminMetrics implements VespaMetrics { // System metrics - CPU_UTIL("cpu.util", Unit.FRACTION, "CPU utilisation"), - CPU_SYS_UTIL("cpu.sys.util", Unit.FRACTION, "System CPU utilisation"), - CPU_THROTTLED_TIME("cpu.throttled_time.rate", Unit.FRACTION, "Part of the time CPU is exhausted (CPU throttling enforced)"), - CPU_THROTTLED_CPU_TIME("cpu.throttled_cpu_time.rate", Unit.FRACTION, "Part of the time CPU is exhausted (CPU throttling enforced)"), + CPU_UTIL("cpu.util", Unit.PERCENTAGE, "CPU utilisation"), + CPU_SYS_UTIL("cpu.sys.util", Unit.PERCENTAGE, "System CPU utilisation"), + CPU_THROTTLED_TIME("cpu.throttled_time.rate", Unit.PERCENTAGE, "Part of the time CPU is exhausted (CPU throttling enforced)"), + CPU_THROTTLED_CPU_TIME("cpu.throttled_cpu_time.rate", Unit.PERCENTAGE, "Part of the time CPU is exhausted (CPU throttling enforced)"), CPU_VCPUS("cpu.vcpus", Unit.ITEM, "Number of virtual CPU threads allocation to the node"), DISK_LIMIT("disk.limit", Unit.BYTE, "Amount of disk space available on the node"), DISK_USED("disk.used", Unit.BYTE, "Amount of disk space used by the node"), - DISK_UTIL("disk.util", Unit.FRACTION, "Disk space utilisation"), + DISK_UTIL("disk.util", Unit.PERCENTAGE, "Disk space utilisation"), MEM_LIMIT("mem.limit", Unit.BYTE, "Amount of memory available on the node"), MEM_USED("mem.used", Unit.BYTE, "Amount of memory used by the node"), - MEM_UTIL("mem.util", Unit.FRACTION, "Memory utilisation"), + MEM_UTIL("mem.util", Unit.PERCENTAGE, "Memory utilisation"), MEM_TOTAL_USED("mem_total.used", Unit.BYTE, "Total amount of memory used by the node, including OS buffer caches"), - MEM_TOTAL_UTIL("mem_total.util", Unit.FRACTION, "Total memory utilisation"), - GPU_UTIL("gpu.util", Unit.FRACTION, "GPU utilisation"), + MEM_TOTAL_UTIL("mem_total.util", Unit.PERCENTAGE, "Total memory utilisation"), + GPU_UTIL("gpu.util", Unit.PERCENTAGE, "GPU utilisation"), GPU_MEM_USED("gpu.memory.used", Unit.BYTE, "GPU memory used"), GPU_MEM_TOTAL("gpu.memory.total", Unit.BYTE, "GPU memory available"), diff --git a/container-core/src/main/java/com/yahoo/metrics/Unit.java b/container-core/src/main/java/com/yahoo/metrics/Unit.java index 7411b5b0ca4..c79461a191f 100644 --- a/container-core/src/main/java/com/yahoo/metrics/Unit.java +++ b/container-core/src/main/java/com/yahoo/metrics/Unit.java @@ -26,6 +26,7 @@ public enum Unit { PACKET(BaseUnit.PACKET), OPERATION(BaseUnit.OPERATION), OPERATION_PER_SECOND(BaseUnit.OPERATION, BaseUnit.SECOND), + PERCENTAGE(BaseUnit.PERCENTAGE), QUERY(BaseUnit.QUERY), QUERY_PER_SECOND(BaseUnit.QUERY, BaseUnit.SECOND), RECORD(BaseUnit.RECORD), @@ -85,6 +86,7 @@ public enum Unit { NODE("node"), OPERATION("operation"), PACKET("packet"), + PERCENTAGE("percentage"), QUERY("query"), RECORD("record"), REQUEST("request"), diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index 8c154072a42..4f770ef6add 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -67,6 +67,7 @@ public abstract class Item implements Cloneable { } // These must match the definitions in searchlib/src/vespa/searchlib/parsequery/item_creator.h + @Deprecated // TODO: Remove on Vespa 9 public enum ItemCreator { ORIG(0), @@ -141,9 +142,8 @@ public abstract class Item implements Cloneable { public abstract String getName(); /** - * Sets whether this is a filter term. - * This indicates that the term origins from the filter parameter in the search API. - * The search backend does not handle filter terms any different than non-filter terms. + * Sets whether this is a filter term. Filter terms are treated just like other terms except that + * they will not be highlighted in dynamic summaries. */ public void setFilter(boolean filter) { if (filter) { @@ -158,12 +158,22 @@ public abstract class Item implements Cloneable { return creator == ItemCreator.FILTER; } - /** Returns the item creator value. */ + /** + * Returns the item creator value. + * + * @deprecated use isFilter(boolean) + */ + @Deprecated // TODO: Remove on Vespa 9 public ItemCreator getCreator() { return creator; } - /** Sets the item creator value. */ + /** + * Sets the item creator value. + * + * @deprecated use setFilter(boolean) + */ + @Deprecated // TODO: Remove on Vespa 9 public void setCreator(ItemCreator creator) { this.creator = creator; } diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java index 98a8e5af00d..2c7d78c0671 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java @@ -100,22 +100,17 @@ public class RecallSearcher extends Searcher { * Returns the first word item contained in the given item tree that is an instance of {@link WordItem} with the * given word value. * - * @param root The root of the tree to check. - * @param value The word to look for. - * @return The first node found. + * @param root the root of the tree to check + * @param value the word to look for + * @return the first node found */ private static WordItem findOrigWordItem(Item root, String value) { Deque<Item> stack = new ArrayDeque<>(); stack.push(root); while (!stack.isEmpty()) { Item item = stack.pop(); - if (item.getCreator() == Item.ItemCreator.ORIG && - item instanceof WordItem word) - { - if (word.getWord().equals(value)) { - return word; - } - } + if (! item.isFilter() && item instanceof WordItem word && word.getWord().equals(value)) + return word; if (item instanceof CompositeItem lst) { for (Iterator<Item> it = lst.getItemIterator(); it.hasNext();) { stack.push(it.next()); @@ -128,16 +123,15 @@ public class RecallSearcher extends Searcher { /** * Marks all filter terms in the given query tree as unranked. * - * @param root The root of the tree to update. + * @param root the root of the tree to update */ private static void updateFilterTerms(Item root) { Deque<Item> stack = new ArrayDeque<>(); stack.push(root); while (!stack.isEmpty()) { Item item = stack.pop(); - if (item.getCreator() == Item.ItemCreator.FILTER) { + if (item.isFilter()) item.setRanked(false); - } if (item instanceof CompositeItem lst) { for (Iterator<Item> it = lst.getItemIterator(); it.hasNext();) { stack.push(it.next()); diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index 9bebaf98665..6aa262ca49e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -375,7 +375,7 @@ public class StemmingSearcher extends Searcher { private void copyAttributes(Item blockAsItem, Item replacement) { copyWeight(blockAsItem, replacement); - replacement.setCreator(blockAsItem.getCreator()); + replacement.setFilter(blockAsItem.isFilter()); replacement.setRanked(blockAsItem.isRanked()); replacement.setPositionData(blockAsItem.usePositionData()); } diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/RangeQueryOptimizer.java b/container-search/src/main/java/com/yahoo/search/querytransform/RangeQueryOptimizer.java index 55a9afa5531..152f91b5f9a 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/RangeQueryOptimizer.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/RangeQueryOptimizer.java @@ -61,8 +61,7 @@ public class RangeQueryOptimizer extends Searcher { List<FieldRange> fieldRanges = null; for (Iterator<Item> i = and.getItemIterator(); i.hasNext(); ) { Item item = i.next(); - if ( ! (item instanceof IntItem)) continue; - IntItem intItem = (IntItem)item; + if ( ! (item instanceof IntItem intItem)) continue; if (intItem.getHitLimit() != 0) continue; // each such op gets a different partial set: Cannot be optimized if (intItem.getFromLimit().equals(intItem.getToLimit())) continue; // don't optimize searches for single numbers if (indexFacts.getIndex(intItem.getIndexName()).isMultivalue()) continue; // May match different values in each range @@ -96,6 +95,7 @@ public class RangeQueryOptimizer extends Searcher { } /** Represents the ranges searched in a single field */ + @SuppressWarnings("deprecation") private static final class FieldRange { private Range range = new Range(new Limit(Double.NEGATIVE_INFINITY, false), new Limit(Double.POSITIVE_INFINITY, false)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java index 4e6eda9f0fa..c861d522818 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java @@ -21,8 +21,6 @@ import java.util.logging.Logger; */ public abstract class ControllerMaintainer extends Maintainer { - protected static final Logger log = Logger.getLogger(ControllerMaintainer.class.getName()); - private final Controller controller; /** The systems in which this maintainer should run */ diff --git a/screwdriver.yaml b/screwdriver.yaml index 16f4c9ac9f4..c9de232d893 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -296,17 +296,8 @@ jobs: - verify-container-image: | # Trick to be able to use the documentation testing to verify the image built locally buildah tag docker.io/$IMAGE_NAME:$VESPA_VERSION vespaengine/vespa:latest - # Clone and setup doc tests - git clone -q --depth 1 https://github.com/vespa-engine/documentation - cd documentation - python3 -m pip install -qqq -r test/requirements.txt --user - echo -e "urls:\n - en/vespa-quick-start.html" > test/_quick-start.yaml - # Get the required vespa CLI - VESPA_CLI_VERSION=$(curl -fsSL https://api.github.com/repos/vespa-engine/vespa/releases/latest | grep -Po '"tag_name": "v\K.*?(?=")') && \ - curl -fsSL https://github.com/vespa-engine/vespa/releases/download/v${VESPA_CLI_VERSION}/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64.tar.gz | tar -zxf - -C /opt && \ - ln -sf /opt/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64/bin/vespa /usr/local/bin/ - # Run test - test/test.py -c test/_quick-start.yaml + # Run quick start guide + $SD_SOURCE_DIR/screwdriver/test-quick-start-guide.sh - publish-test-image: | if [[ -z $SD_PULL_REQUEST ]]; then if curl -fsSL https://index.docker.io/v1/repositories/$IMAGE_NAME/tags/$VESPA_VERSION &> /dev/null; then diff --git a/screwdriver/release-container-image.sh b/screwdriver/release-container-image.sh index 6d8babe3dcc..a2a56313fb5 100755 --- a/screwdriver/release-container-image.sh +++ b/screwdriver/release-container-image.sh @@ -61,6 +61,11 @@ for data in "Dockerfile vespa"; do if curl -fsSL https://index.docker.io/v1/repositories/vespaengine/$IMAGE_NAME/tags/$VESPA_VERSION &> /dev/null; then echo "Container image docker.io/vespaengine/$IMAGE_NAME:$VESPA_VERSION aldready exists." else + # Build the image and run through the quick start guide + docker buildx build --progress plain --load --platform linux/amd64 --build-arg VESPA_VERSION=$VESPA_VERSION \ + --file $DOCKER_FILE --tag vespaengine/$IMAGE_NAME:latest . + $SD_SOURCE_DIR/screwdriver/test-quick-start-guide.sh + docker login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" docker buildx build --progress plain --push --platform linux/amd64,linux/arm64 --build-arg VESPA_VERSION=$VESPA_VERSION \ --file $DOCKER_FILE --tag docker.io/vespaengine/$IMAGE_NAME:$VESPA_VERSION \ diff --git a/screwdriver/test-quick-start-guide.sh b/screwdriver/test-quick-start-guide.sh new file mode 100755 index 00000000000..f3f12c229b0 --- /dev/null +++ b/screwdriver/test-quick-start-guide.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# This test will test the quick start guide that use the vespaengine/vespa:latest image. To test a locally +# generated image, make sure that the image is tagged as vespaengine/vespa:latest. +set -xeuo pipefail + +TESTDIR=$(mktemp -d) +trap "rm -rf $TESTDIR" EXIT + +cd $TESTDIR + +# Clone and setup doc tests +git clone -q --depth 1 https://github.com/vespa-engine/documentation +cd documentation +python3 -m pip install -qqq -r test/requirements.txt --user +echo -e "urls:\n - en/vespa-quick-start.html" > test/_quick-start.yaml + +# Get the required vespa CLI +VESPA_CLI_VERSION=$(curl -fsSL https://api.github.com/repos/vespa-engine/vespa/releases/latest | grep -Po '"tag_name": "v\K.*?(?=")') +curl -fsSL https://github.com/vespa-engine/vespa/releases/download/v${VESPA_CLI_VERSION}/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64.tar.gz | tar -zxf - -C /opt +ln -sf /opt/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64/bin/vespa /usr/local/bin/ + +# Run test +test/test.py -v -c test/_quick-start.yaml + diff --git a/vespalib/src/tests/coro/active_work/active_work_test.cpp b/vespalib/src/tests/coro/active_work/active_work_test.cpp index 26c0c3dd71a..6404705fff5 100644 --- a/vespalib/src/tests/coro/active_work/active_work_test.cpp +++ b/vespalib/src/tests/coro/active_work/active_work_test.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/gtest/gtest.h> +#include <cinttypes> using namespace vespalib; using namespace vespalib::coro; @@ -49,7 +50,7 @@ TEST(ActiveWorkTest, run_expensive_subtasks_concurrently) { make_expensive_task)); auto td = steady_clock::now() - t0; EXPECT_EQ(result, 136); - fprintf(stderr, "time spent: %zu ms\n", count_ms(td)); + fprintf(stderr, "time spent: %" PRId64 " ms\n", count_ms(td)); } TEST(ActiveWorkTest, run_cheap_subtasks_concurrently) { @@ -60,7 +61,7 @@ TEST(ActiveWorkTest, run_cheap_subtasks_concurrently) { make_cheap_task)); auto td = steady_clock::now() - t0; EXPECT_EQ(result, 136); - fprintf(stderr, "time spent: %zu ms\n", count_ms(td)); + fprintf(stderr, "time spent: %" PRId64 " ms\n", count_ms(td)); } GTEST_MAIN_RUN_ALL_TESTS() |