diff options
19 files changed, 221 insertions, 191 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java deleted file mode 100644 index 5a7f2db1968..00000000000 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.model.application.provider; - -import com.yahoo.config.FileReference; -import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.net.HostName; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.Reader; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; - -/** - * Registry of files added earlier (i.e. during deployment) - * - * @author Tony Vaagenes - */ -public class PreGeneratedFileRegistry implements FileRegistry { - - private final String fileSourceHost; - private final Map<String, FileReference> path2Hash; - - private static final String entryDelimiter = "\t"; - private static final Pattern entryDelimiterPattern = Pattern.compile(entryDelimiter, Pattern.LITERAL); - - public static Map<String, FileReference> decode(BufferedReader reader) { - Map<String, FileReference> refs = new HashMap<>(); - try { - String line; - while ((line = reader.readLine()) != null) { - String[] parts = entryDelimiterPattern.split(line); - if (parts.length < 2) - throw new IllegalArgumentException("Cannot split '" + line + "' into two parts"); - refs.put(parts[0], new FileReference(parts[1])); - } - } catch (IOException e) { - throw new RuntimeException("Error while reading pre-generated file registry", e); - } - return refs; - } - private PreGeneratedFileRegistry(Reader readerArg) { - try (BufferedReader reader = new BufferedReader(readerArg)) { - fileSourceHost = reader.readLine(); - if (fileSourceHost == null) - throw new RuntimeException("Error while reading pre-generated file registry"); - - path2Hash = decode(reader); - } catch (IOException e) { - throw new RuntimeException("Error while reading pre-generated file registry", e); - } - } - - public static String exportRegistry(FileRegistry registry) { - List<Entry> entries = registry.export(); - StringBuilder builder = new StringBuilder(); - - builder.append(HostName.getLocalhost()).append('\n'); - for (FileRegistry.Entry entry : entries) { - builder.append(entry.relativePath).append(entryDelimiter).append(entry.reference.value()).append('\n'); - } - - return builder.toString(); - } - - public static PreGeneratedFileRegistry importRegistry(Reader reader) { - return new PreGeneratedFileRegistry(reader); - } - - public FileReference addFile(String relativePath) { - FileReference reference = path2Hash.get(relativePath); - if (reference == null) { - throw new IllegalArgumentException("File '" + relativePath + "' not found"); - } - return reference; - } - - @Override - public FileReference addUri(String uri) { - FileReference reference = path2Hash.get(uri); - if (reference == null) { - throw new IllegalArgumentException("Uri '" + uri + "' not found"); - } - return reference; - } - @Override - public FileReference addBlob(ByteBuffer blob) { - String blobName = FileRegistry.blobName(blob); - FileReference reference = path2Hash.get(blobName); - if (reference == null) { - throw new IllegalArgumentException("Blob '" + blobName + "(" + blob.remaining()+ ")' not found"); - } - return reference; - } - - public Set<String> getPaths() { - return path2Hash.keySet(); - } - - @Override - public List<Entry> export() { - List<Entry> entries = new ArrayList<>(); - for (Map.Entry<String, FileReference> entry : path2Hash.entrySet()) { - entries.add(new Entry(entry.getKey(), entry.getValue())); - } - return entries; - } -} diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistryTestCase.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistryTestCase.java deleted file mode 100644 index 532ebb1ab81..00000000000 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistryTestCase.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.model.application.provider; - -import com.yahoo.config.application.api.FileRegistry; -import org.junit.Test; - -import java.io.StringReader; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.Set; - -import static org.junit.Assert.assertEquals; - -/** - * @author Tony Vaagenes - */ -public class PreGeneratedFileRegistryTestCase { - - private static final String BLOB = "Some blob"; - @Test - public void importAndExport() { - FileRegistry fileRegistry = new MockFileRegistry(); - fileRegistry.addFile("1234"); - fileRegistry.addBlob(ByteBuffer.wrap(BLOB.getBytes(StandardCharsets.UTF_8))); - String serializedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry); - - PreGeneratedFileRegistry importedRegistry = PreGeneratedFileRegistry.importRegistry(new StringReader(serializedRegistry)); - - assertEquals(Set.of("1234", "c5674b55c15c9c95.blob"), importedRegistry.getPaths()); - - assertEquals(2, importedRegistry.getPaths().size()); - - checkConsistentEntry(fileRegistry.export().get(0), importedRegistry); - checkConsistentEntry(fileRegistry.export().get(1), importedRegistry); - } - - void checkConsistentEntry(FileRegistry.Entry entry, FileRegistry registry) { - assertEquals(entry.reference, registry.addFile(entry.relativePath)); - } -} diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java index 7e4a6aec53a..a0b1402f559 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java @@ -39,7 +39,9 @@ public interface FileRegistry { } static String blobName(ByteBuffer blob) { + blob.mark(); long blobHash = XXHashFactory.fastestJavaInstance().hash64().hash(blob, 0); + blob.reset(); return Long.toHexString(blobHash); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 912efbd9a24..ec5da9baa11 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -471,6 +471,7 @@ public class VespaMetricSet { metrics.add(new Metric("content.proton.resource_usage.open_file_descriptors.max")); metrics.add(new Metric("content.proton.resource_usage.feeding_blocked.max")); metrics.add(new Metric("content.proton.resource_usage.malloc_arena.max")); + metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.address_space.max")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.enum_store.average")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.multi_value.average")); metrics.add(new Metric("content.proton.documentdb.attribute.resource_usage.feeding_blocked.last")); // TODO: Remove in Vespa 8 diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java index 99632ec323e..bb99771b475 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ZooKeeperClient.java @@ -8,13 +8,13 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; -import com.yahoo.config.model.application.provider.PreGeneratedFileRegistry; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.serialization.AllocatedHostsSerializer; import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigDefinitionKey; +import com.yahoo.vespa.config.server.filedistribution.FileDBRegistry; import com.yahoo.vespa.config.server.zookeeper.ZKApplicationPackage; import com.yahoo.vespa.curator.Curator; import com.yahoo.yolean.Exceptions; @@ -228,7 +228,7 @@ public class ZooKeeperClient { } private void write(Version vespaVersion, FileRegistry fileRegistry) { - String exportedRegistry = PreGeneratedFileRegistry.exportRegistry(fileRegistry); + String exportedRegistry = FileDBRegistry.exportRegistry(fileRegistry); curator.set(getZooKeeperAppPath(ZKApplicationPackage.fileRegistryNode).append(vespaVersion.toFullString()), Utf8.toBytes(exportedRegistry)); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java index 163c19abe75..1218ef6599d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; +import java.io.IOException; import java.nio.ByteBuffer; /** @@ -10,6 +11,6 @@ import java.nio.ByteBuffer; */ public interface AddFileInterface { FileReference addUri(String uri, String relativePath); - FileReference addFile(String relativePath); + FileReference addFile(String relativePath) throws IOException; FileReference addBlob(ByteBuffer blob, String relativePath); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java index a1907c01085..064cb6423da 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java @@ -26,20 +26,28 @@ public class ApplicationFileManager implements AddFileInterface { } @Override - public FileReference addFile(String relativePath) { + public FileReference addFile(String relativePath) throws IOException { return fileDirectory.addFile(new File(applicationDir, relativePath)); } @Override public FileReference addUri(String uri, String relativePath) { download(uri, relativePath); - return addFile(relativePath); + try { + return addFile(relativePath); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } } @Override public FileReference addBlob(ByteBuffer blob, String relativePath) { writeBlob(blob, relativePath); - return addFile(relativePath); + try { + return addFile(relativePath); + } catch (IOException e) { + throw new IllegalArgumentException(e); + } } private void writeBlob(ByteBuffer blob, String relativePath) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java index 5ec83529601..0d0f6d09582 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.filedistribution; +import com.google.common.collect.ImmutableMap; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.net.HostName; @@ -8,6 +9,7 @@ import com.yahoo.text.Utf8; import net.jpountz.xxhash.XXHashFactory; import java.io.BufferedReader; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.Reader; import java.nio.ByteBuffer; @@ -23,15 +25,17 @@ import java.util.regex.Pattern; */ public class FileDBRegistry implements FileRegistry { + private final boolean silenceNonExistingFiles; private final AddFileInterface manager; private final Map<String, FileReference> fileReferenceCache = new HashMap<>(); private static final String entryDelimiter = "\t"; private static final Pattern entryDelimiterPattern = Pattern.compile(entryDelimiter, Pattern.LITERAL); public FileDBRegistry(AddFileInterface manager) { + silenceNonExistingFiles = false; this.manager = manager; } - public static FileRegistry create(AddFileInterface manager, Reader persistedState) { + public static FileDBRegistry create(AddFileInterface manager, Reader persistedState) { try (BufferedReader reader = new BufferedReader(persistedState)) { String ignoredFileSourceHost = reader.readLine(); if (ignoredFileSourceHost == null) @@ -58,20 +62,30 @@ public class FileDBRegistry implements FileRegistry { return refs; } private FileDBRegistry(AddFileInterface manager, Map<String, FileReference> knownReferences) { + silenceNonExistingFiles = true; this.manager = manager; - for (Map.Entry<String, FileReference> e : knownReferences.entrySet()) { - fileReferenceCache.put(e.getKey(), e.getValue()); - } + fileReferenceCache.putAll(knownReferences); } @Override public synchronized FileReference addFile(String relativePath) { Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath)); return cachedReference.orElseGet(() -> { - FileReference newRef = manager.addFile(relativePath); - fileReferenceCache.put(relativePath, newRef); - return newRef; - }); + try { + FileReference newRef = manager.addFile(relativePath); + fileReferenceCache.put(relativePath, newRef); + return newRef; + } catch (FileNotFoundException e) { + if (silenceNonExistingFiles) { + return new FileReference("non-existing-file"); + } else { + throw new IllegalArgumentException(e); + } + } catch (IOException e) { + throw new IllegalArgumentException(e); + } + } + ); } @Override @@ -88,7 +102,7 @@ public class FileDBRegistry implements FileRegistry { @Override public FileReference addBlob(ByteBuffer blob) { String blobName = FileRegistry.blobName(blob); - String relativePath = blobToRelativeFile(blob, blobName); + String relativePath = blobToRelativeFile(blobName); synchronized (this) { Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(blobName)); return cachedReference.orElseGet(() -> { @@ -108,6 +122,22 @@ public class FileDBRegistry implements FileRegistry { return entries; } + synchronized Map<String, FileReference> getMap() { + return ImmutableMap.copyOf(fileReferenceCache); + } + + public static String exportRegistry(FileRegistry registry) { + List<Entry> entries = registry.export(); + StringBuilder builder = new StringBuilder(); + + builder.append(HostName.getLocalhost()).append('\n'); + for (FileRegistry.Entry entry : entries) { + builder.append(entry.relativePath).append(entryDelimiter).append(entry.reference.value()).append('\n'); + } + + return builder.toString(); + } + private static String uriToRelativeFile(String uri) { String relative = "uri/" + XXHashFactory.fastestJavaInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0); if (uri.endsWith(".json")) { @@ -120,9 +150,8 @@ public class FileDBRegistry implements FileRegistry { return relative; } - private static String blobToRelativeFile(ByteBuffer blob, String blobName) { - String relative = "blob/" + blobName; - return relative; + private static String blobToRelativeFile(String blobName) { + return "blob/" + blobName; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index f7ba895666b..1fe8c741a17 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -97,15 +97,11 @@ public class FileDirectory { return hasher.hash(ByteBuffer.wrap(wholeFile), hasher.hash(ByteBuffer.wrap(Utf8.toBytes(file.getName())), 0)); } - public FileReference addFile(File source) { - try { - Long hash = computeHash(source); - verifyExistingFile(source, hash); - FileReference fileReference = fileReferenceFromHash(hash); - return addFile(source, fileReference); - } catch (IOException e) { - throw new IllegalArgumentException(e); - } + public FileReference addFile(File source) throws IOException { + Long hash = computeHash(source); + verifyExistingFile(source, hash); + FileReference fileReference = fileReferenceFromHash(hash); + return addFile(source, fileReference); } // If there exists a directory for a file reference, but content does not have correct hash or the file we want to add diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java new file mode 100644 index 00000000000..10ac54dc757 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistryTestCase.java @@ -0,0 +1,66 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.filedistribution; + +import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.FileRegistry; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * @author Tony Vaagenes + */ +public class FileDBRegistryTestCase { + + private static final String BLOB = "Some blob"; + private static final String APP = "src/test/apps/zkapp"; + private static final String FOO_FILE = "files/foo.json"; + private static final String NO_FOO_FILE = "files/no_foo.json"; + private static final String BLOB_NAME = "c5674b55c15c9c95"; + private static final FileReference BLOB_REF = new FileReference("b3424b78130fc005"); + private static final FileReference FOO_REF = new FileReference("b5ce94ca1feae86c"); + @Test + public void importAndExport() throws IOException { + TemporaryFolder tmpDir = new TemporaryFolder(); + tmpDir.create(); + AddFileInterface fileManager = new ApplicationFileManager(new File(APP), new FileDirectory(tmpDir.newFolder())); + FileRegistry fileRegistry = new FileDBRegistry(fileManager); + assertEquals(FOO_REF, fileRegistry.addFile(FOO_FILE)); + try { + fileRegistry.addFile(NO_FOO_FILE); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("src/test/apps/zkapp/files/no_foo.json (No such file or directory)", e.getCause().getMessage()); + } + assertEquals(BLOB_REF, fileRegistry.addBlob(ByteBuffer.wrap(BLOB.getBytes(StandardCharsets.UTF_8)))); + String serializedRegistry = FileDBRegistry.exportRegistry(fileRegistry); + + FileDBRegistry importedRegistry = FileDBRegistry.create(fileManager, new StringReader(serializedRegistry)); + + assertEquals(Set.of(BLOB_NAME, FOO_FILE), importedRegistry.getMap().keySet()); + assertEquals(BLOB_REF, importedRegistry.getMap().get(BLOB_NAME)); + assertEquals(FOO_REF, importedRegistry.getMap().get(FOO_FILE)); + + assertEquals(2, importedRegistry.export().size()); + + checkConsistentEntry(fileRegistry.export().get(0), importedRegistry); + checkConsistentEntry(fileRegistry.export().get(1), importedRegistry); + + assertEquals(new FileReference("non-existing-file"), importedRegistry.addFile(NO_FOO_FILE)); + assertEquals(2, importedRegistry.export().size()); + tmpDir.delete(); + } + + void checkConsistentEntry(FileRegistry.Entry entry, FileRegistry registry) { + assertEquals(entry.reference, registry.addFile(entry.relativePath)); + } +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java index b11308214a3..8b3227e3a69 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/filedistribution/MockFileRegistry.java @@ -3,10 +3,10 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.FileRegistry; -import com.yahoo.net.HostName; import net.jpountz.xxhash.XXHashFactory; import java.io.File; +import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.ArrayList; @@ -30,11 +30,16 @@ public class MockFileRegistry implements FileRegistry { public FileReference addFile(String relativePath) { if (relativePath.isEmpty()) relativePath = "./"; - addFileInterface.addFile(relativePath); - FileReference fileReference = new FileReference(relativePath); - entries.add(new Entry(relativePath, fileReference)); - return fileReference; + try { + addFileInterface.addFile(relativePath); + + FileReference fileReference = new FileReference(relativePath); + entries.add(new Entry(relativePath, fileReference)); + return fileReference; + } catch (IOException e) { + throw new IllegalArgumentException(e); + } } public List<Entry> export() { return entries; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json index 2cf846ab6bb..0525f059dd0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-test-log.json @@ -59,6 +59,21 @@ "at": 14503000, "type": "info", "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": 14503000, + "type": "info", + "message": "host-tenant:application:default-t-staging.us-east-3: unorchestrated" + }, + { + "at": 14503000, + "type": "info", + "message": "--- platform vespa/vespa:6.1" + }, + { + "at": 14503000, + "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" } ], "deployInitialReal": [ @@ -102,6 +117,21 @@ { "at": 14503000, "type": "info", + "message": "host-tenant:application:default-staging.us-east-3: unorchestrated" + }, + { + "at": 14503000, + "type": "info", + "message": "--- platform vespa/vespa:6.1" + }, + { + "at": 14503000, + "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": 14503000, + "type": "info", "message": "Deployment expired before installation was successful." } ], @@ -120,7 +150,7 @@ } ] }, - "lastId": 21, + "lastId": 27, "steps": { "deployTester": { "status": "succeeded", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json index 2f5e93ea3a0..7f59eaf75c2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-log.json @@ -73,6 +73,21 @@ { "at": 0, "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": 0, + "type": "info", + "message": "host-tenant:application:default-t-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "info", + "message": "--- platform vespa/vespa:6.1" + }, + { + "at": 0, + "type": "info", "message": "Tester container successfully installed!" } ], @@ -192,6 +207,21 @@ { "at": 0, "type": "info", + "message": "host-tenant:application:default-test.us-east-1: unorchestrated" + }, + { + "at": 0, + "type": "info", + "message": "--- platform vespa/vespa:6.1" + }, + { + "at": 0, + "type": "info", + "message": "--- container on port 43 has config generation 1, wanted is 2" + }, + { + "at": 0, + "type": "info", "message": "Found endpoints:" }, { @@ -259,7 +289,7 @@ } ] }, - "lastId": 48, + "lastId": 54, "steps": { "deployTester": { "status": "succeeded", diff --git a/screwdriver/build-vespa.sh b/screwdriver/build-vespa.sh index 91375728ca9..46528bc301c 100755 --- a/screwdriver/build-vespa.sh +++ b/screwdriver/build-vespa.sh @@ -69,7 +69,7 @@ if [[ $SHOULD_BUILD == systemtest ]]; then export SYSTEM_TEST_DIR=$(pwd)/system-test export RUBYLIB="$SYSTEM_TEST_DIR/lib:$SYSTEM_TEST_DIR/tests" useradd vespa - export USER=$(whoami) + export USER=vespa $SYSTEM_TEST_DIR/lib/node_server.rb & NODE_SERVER_PID=$! diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index db76d8bad25..f577aedfbf1 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -716,8 +716,14 @@ Fixture::test_populate_address_space_usage() { search::AddressSpaceUsage usage = _attr->getAddressSpaceUsage(); const auto& all = usage.get_all(); - EXPECT_EQUAL(1u, all.size()); - EXPECT_EQUAL(1u, all.count("tensor-store")); + if (_denseTensors) { + EXPECT_EQUAL(1u, all.size()); + EXPECT_EQUAL(1u, all.count("tensor-store")); + } else { + EXPECT_EQUAL(2u, all.size()); + EXPECT_EQUAL(1u, all.count("tensor-store")); + EXPECT_EQUAL(1u, all.count("shared-string-repo")); + } } template <class MakeFixture> diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp index 1e6ad8f4ef7..00f2ea4a0a3 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.cpp @@ -18,6 +18,7 @@ AddressSpace AddressSpaceComponents::default_multi_value_usage() { const vespalib::string AddressSpaceComponents::enum_store = "enum-store"; const vespalib::string AddressSpaceComponents::multi_value = "multi-value"; const vespalib::string AddressSpaceComponents::tensor_store = "tensor-store"; +const vespalib::string AddressSpaceComponents::shared_string_repo = "shared-string-repo"; const vespalib::string AddressSpaceComponents::hnsw_node_store = "hnsw-node-store"; const vespalib::string AddressSpaceComponents::hnsw_link_store = "hnsw-link-store"; diff --git a/searchlib/src/vespa/searchlib/attribute/address_space_components.h b/searchlib/src/vespa/searchlib/attribute/address_space_components.h index 213d86fa803..257d3079f1a 100644 --- a/searchlib/src/vespa/searchlib/attribute/address_space_components.h +++ b/searchlib/src/vespa/searchlib/attribute/address_space_components.h @@ -17,6 +17,7 @@ public: static const vespalib::string enum_store; static const vespalib::string multi_value; static const vespalib::string tensor_store; + static const vespalib::string shared_string_repo; static const vespalib::string hnsw_node_store; static const vespalib::string hnsw_link_store; }; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index d61fc544839..8ecce17f033 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/data/slime/inserter.h> #include <vespa/vespalib/util/rcuvector.hpp> +#include <vespa/vespalib/util/shared_string_repo.h> #include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/tensor_spec.h> @@ -53,6 +54,7 @@ TensorAttribute::TensorAttribute(vespalib::stringref name, const Config &cfg, Te cfg.getGrowStrategy().getDocsGrowDelta(), getGenerationHolder()), _tensorStore(tensorStore), + _is_dense(cfg.tensorType().is_dense()), _emptyTensor(createEmptyTensor(cfg.tensorType())), _compactGeneration(0), _cached_tensor_store_memory_usage() @@ -195,6 +197,11 @@ void TensorAttribute::populate_address_space_usage(AddressSpaceUsage& usage) const { usage.set(AddressSpaceComponents::tensor_store, _tensorStore.get_address_space_usage()); + if (!_is_dense) { + auto stats = vespalib::SharedStringRepo::stats(); + usage.set(AddressSpaceComponents::shared_string_repo, + vespalib::AddressSpace(stats.max_part_usage, 0, stats.part_limit())); + } } vespalib::eval::Value::UP diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index 115a8d34058..156a04bdb14 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -24,6 +24,7 @@ protected: RefVector _refVector; // docId -> ref in data store for serialized tensor TensorStore &_tensorStore; // data store for serialized tensors + bool _is_dense; std::unique_ptr<vespalib::eval::Value> _emptyTensor; uint64_t _compactGeneration; // Generation when last compact occurred vespalib::MemoryUsage _cached_tensor_store_memory_usage; |