diff options
author | Harald Musum <musum@yahoo-inc.com> | 2018-01-29 07:49:55 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-01-29 07:49:55 +0100 |
commit | 36a9d6c30950aafaa235a9a9de8d53283d351914 (patch) | |
tree | 20a6e72bf6fb3513268a71dec22cdc4f552d0e45 | |
parent | c73689030686fc239e2854bc77d07027df7c6e81 (diff) | |
parent | 116a1fa10dd3c5f98fc4d6e8cdcf762fe79df05f (diff) |
Merge pull request #4785 from vespa-engine/balder/add-uri-support-to-tensor-constants-3
Balder/add uri support to tensor constants 3
18 files changed, 290 insertions, 19 deletions
diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java index d635fe90ded..515477641a8 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/MockFileRegistry.java @@ -35,4 +35,9 @@ public class MockFileRegistry implements FileRegistry { return result; } + @Override + public FileReference addUri(String uri) { + throw new IllegalArgumentException("FileReference addUri(String uri) is not implemented for " + getClass().getCanonicalName()); + } + } diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java index 0b0b799f47f..ed85b987a3d 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/PreGeneratedFileRegistry.java @@ -74,6 +74,11 @@ public class PreGeneratedFileRegistry implements FileRegistry { } @Override + public FileReference addUri(String uri) { + return new FileReference(path2Hash.get(uri)); + } + + @Override public String fileSourceHost() { return fileSourceHost; } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java index 887da2d51c8..15ae4294762 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/FileRegistry.java @@ -2,16 +2,17 @@ package com.yahoo.config.application.api; import java.util.List; -import java.util.Set; import com.yahoo.config.FileReference; + /** * @author tonytv */ public interface FileRegistry { FileReference addFile(String relativePath); + FileReference addUri(String uri); /** * Returns the name of the host which is the source of the files diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java index c65e0fad1c7..a2bdc6834c9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java @@ -12,12 +12,20 @@ import java.util.Objects; */ public class RankingConstant { + public enum PathType {FILE, URI}; + /** The search definition-unique name of this constant */ private final String name; private TensorType tensorType = null; - private String fileName = null; + private String path = null; private String fileReference = ""; + public PathType getPathType() { + return pathType; + } + + private PathType pathType = PathType.FILE; + public RankingConstant(String name) { this.name = name; } @@ -25,13 +33,20 @@ public class RankingConstant { public RankingConstant(String name, TensorType type, String fileName) { this(name); this.tensorType = type; - this.fileName = fileName; + this.path = fileName; validate(); } public void setFileName(String fileName) { Objects.requireNonNull(fileName, "Filename cannot be null"); - this.fileName = fileName; + this.path = fileName; + this.pathType = PathType.FILE; + } + + public void setUri(String uri) { + Objects.requireNonNull(uri, "uri cannot be null"); + this.path = uri; + this.pathType = PathType.URI; } /** @@ -43,14 +58,15 @@ public class RankingConstant { public void setType(TensorType tensorType) { this.tensorType = tensorType; } public String getName() { return name; } - public String getFileName() { return fileName; } + public String getFileName() { return path; } + public String getUri() { return path; } public String getFileReference() { return fileReference; } public TensorType getTensorType() { return tensorType; } public String getType() { return tensorType.toString(); } public void validate() { - if (fileName == null || fileName.isEmpty()) - throw new IllegalArgumentException("Ranking constants must have a file."); + if (path == null || path.isEmpty()) + throw new IllegalArgumentException("Ranking constants must have a file or uri."); if (tensorType == null) throw new IllegalArgumentException("Ranking constant '" + name + "' must have a type."); } @@ -58,7 +74,7 @@ public class RankingConstant { public String toString() { StringBuilder b = new StringBuilder(); b.append("constant '").append(name) - .append("' from file '").append(fileName) + .append(pathType == PathType.FILE ? "' from file '" : " from uri ").append(path) .append("' with ref '").append(fileReference) .append("' of type '").append(tensorType) .append("'"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 3b75be5167d..65f7bbedc68 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -508,6 +508,13 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon throw new RuntimeException("File does not exist: '" + relativePath + "'."); } } + public FileReference sendUri(String uri) { + try { + return getRoot().getFileDistributor().sendUriToHost(uri, getHost()); + } catch (PathDoesNotExistException e) { + throw new RuntimeException("Uri does not exist: '" + uri + "'."); + } + } /** * diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java index 520ff231921..d022b2cf8ab 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java @@ -63,15 +63,19 @@ public class RankingConstantsValidator extends Validator { } private void validateRankingConstant(RankingConstant rankingConstant, ApplicationPackage application) throws FileNotFoundException { - String constantFile = rankingConstant.getFileName(); - if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) && - constantFile.startsWith(FilesApplicationPackage.preprocessed)) - constantFile = constantFile.substring(FilesApplicationPackage.preprocessed.length()); + // TODO: Handle validation of URI soon too. + if (rankingConstant.getPathType() == RankingConstant.PathType.FILE) { + String constantFile = rankingConstant.getFileName(); + if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) && + constantFile.startsWith(FilesApplicationPackage.preprocessed)) { + constantFile = constantFile.substring(FilesApplicationPackage.preprocessed.length()); + } - ApplicationFile tensorApplicationFile = application.getFile(Path.fromString(constantFile)); - new ConstantTensorJsonValidator().validate(constantFile, - rankingConstant.getTensorType(), - tensorApplicationFile.createReader()); + ApplicationFile tensorApplicationFile = application.getFile(Path.fromString(constantFile)); + new ConstantTensorJsonValidator().validate(constantFile, + rankingConstant.getTensorType(), + tensorApplicationFile.createReader()); + } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java index 213451da55e..e8d6a330358 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/FileDistributor.java @@ -38,11 +38,31 @@ public class FileDistributor { return reference; } + /** + * Adds the given file to the associated application packages' registry of file and marks the file + * for distribution to the given hosts. + * <b>Note: This class receives ownership of the given collection.</b> + * + * @return the reference to the file, created by the application package + */ + public FileReference sendUriToHosts(String uri, Collection<Host> hosts) { + FileReference reference = fileRegistry.addUri(uri); + if (reference != null) { + addToFilesToDistribute(reference, hosts); + } + + return reference; + } + /** Same as sendFileToHost(relativePath,Collections.singletonList(host) */ public FileReference sendFileToHost(String relativePath, Host host) { return sendFileToHosts(relativePath, Arrays.asList(host)); } + public FileReference sendUriToHost(String uri, Host host) { + return sendUriToHosts(uri, Arrays.asList(host)); + } + private void addToFilesToDistribute(FileReference reference, Collection<Host> hosts) { Set<Host> oldHosts = getHosts(reference); oldHosts.addAll(hosts); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java index 58fc76f1508..9550cd82b22 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java @@ -38,7 +38,9 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer public void prepareToDistributeFiles(List<SearchNode> backends) { for (SearchDefinitionSpec sds : localSDS) { for (RankingConstant constant : sds.getSearchDefinition().getSearch().getRankingConstants().values()) { - FileReference reference = FileSender.sendFileToServices(constant.getFileName(), backends); + FileReference reference = (constant.getPathType() == RankingConstant.PathType.FILE) + ? FileSender.sendFileToServices(constant.getFileName(), backends) + : FileSender.sendUriToServices(constant.getUri(), backends); constant.setFileReference(reference.value()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java index 413363d7b0d..8995fcbca99 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/FileSender.java @@ -20,6 +20,8 @@ import java.util.*; */ public class FileSender implements Serializable { + public enum FileType {FILE, URI}; + /** * Send the given file to all given services. * @@ -34,6 +36,7 @@ public class FileSender implements Serializable { throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " + "using 'count' instead of <node> tags."); } + FileReference fileref = null; for (AbstractService service : services) { // The same reference will be returned from each call. @@ -42,6 +45,20 @@ public class FileSender implements Serializable { return fileref; } + public static FileReference sendUriToServices(String uri, Collection<? extends AbstractService> services) { + if (services.isEmpty()) { + throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " + + "using 'count' instead of <node> tags."); + } + + FileReference fileref = null; + for (AbstractService service : services) { + // The same reference will be returned from each call. + fileref = service.sendUri(uri); + } + return fileref; + } + /** * Sends all user configured files for a producer to all given services. */ diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 916a905dfcb..bf6376983a4 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -339,6 +339,7 @@ TOKEN : | < RANKSCOREDROPLIMIT: "rank-score-drop-limit" > | < CONSTANTS: "constants" > | < FILE: "file" > +| < URI: "uri" > | < IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-"])* > | < QUOTEDSTRING: "\"" ( ~["\""] )* "\"" > | < CONTEXT: ["a"-"z","A"-"Z"] (["a"-"z", "A"-"Z", "0"-"9"])* > @@ -347,6 +348,8 @@ TOKEN : | < LONG: ("-")? (["0"-"9"])+"L" > | < STRING: (["a"-"z","A"-"Z","_","0"-"9","."])+ > | < FILE_PATH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-", "/", "."])+ > +| < HTTP: ["h","H"] ["t","T"] ["t","T"] ["p","P"] > +| < URI_PATH: <HTTP> <COLON> ("//")? (["a"-"z","A"-"Z","0"-"9","_","-", "/", ".",":"])+ > | < LESSTHAN: "<" > | < GREATERTHAN: ">" > | < VARIABLE: "$" <IDENTIFIER> > @@ -1805,11 +1808,12 @@ void rankingConstant(Search search) : */ Object rankingConstantItem(RankingConstant constant) : { - String fileName = null; + String path = null; TensorType type = null; } { - ( (<FILE> <COLON> fileName = filePath() { } (<NL>)*) { constant.setFileName(fileName); } + ( (<FILE> <COLON> path = filePath() { } (<NL>)*) { constant.setFileName(path); } + | (<URI> <COLON> path = uriPath() { } (<NL>)*) { constant.setUri(path); } | type = tensorTypeWithPrefix(rankingConstantErrorMessage(constant.getName())) (<NL>)* { constant.setType(type); } ) { @@ -1828,6 +1832,12 @@ String filePath() : { } { return token.image; } } +String uriPath() : { } +{ + ( <URI_PATH> ) + { return token.image; } +} + /** * Consumes a rank-profile block of a search element. * @@ -2550,6 +2560,7 @@ String identifier() : { } | <TRUE> | <TYPE> | <UCA> + | <URI> | <UPPERBOUND> | <USEDOCUMENT> | <VARIABLE> diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java b/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java index 2880af9e74f..9bad5373191 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java @@ -1,6 +1,7 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition; +import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -48,6 +49,7 @@ public class RankingConstantTest { assertEquals(TENSOR_NAME, constant.getName()); assertEquals(TENSOR_FILE, constant.getFileName()); assertEquals(TENSOR_TYPE, constant.getType()); + assertEquals(RankingConstant.PathType.FILE, constant.getPathType()); assertFalse(constantIterator.hasNext()); } @@ -103,4 +105,80 @@ public class RankingConstantTest { assertEquals("simplename", constant.getFileName()); } + @Test + public void constant_uri_is_allowed() throws Exception { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry); + searchBuilder.importString(joinLines( + "search test {", + " document test { }", + " constant foo {", + " type: tensor(x{})", + " uri: http://somwhere.far.away/in/another-galaxy", + " }", + "}" + )); + searchBuilder.build(); + Search search = searchBuilder.getSearch(); + RankingConstant constant = search.getRankingConstants().values().iterator().next(); + assertEquals(RankingConstant.PathType.URI, constant.getPathType()); + assertEquals("http://somwhere.far.away/in/another-galaxy", constant.getUri()); + } + @Test + public void constant_uri_with_port_is_allowed() throws Exception { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry); + searchBuilder.importString(joinLines( + "search test {", + " document test { }", + " constant foo {", + " type: tensor(x{})", + " uri: http://somwhere.far.away:4080/in/another-galaxy", + " }", + "}" + )); + searchBuilder.build(); + Search search = searchBuilder.getSearch(); + RankingConstant constant = search.getRankingConstants().values().iterator().next(); + assertEquals(RankingConstant.PathType.URI, constant.getPathType()); + assertEquals("http://somwhere.far.away:4080/in/another-galaxy", constant.getUri()); + } + @Test + public void constant_uri_no_dual_slashes_is_allowed() throws Exception { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry); + searchBuilder.importString(joinLines( + "search test {", + " document test { }", + " constant foo {", + " type: tensor(x{})", + " uri: http:somwhere.far.away/in/another-galaxy", + " }", + "}" + )); + searchBuilder.build(); + Search search = searchBuilder.getSearch(); + RankingConstant constant = search.getRankingConstants().values().iterator().next(); + assertEquals(RankingConstant.PathType.URI, constant.getPathType()); + assertEquals("http:somwhere.far.away/in/another-galaxy", constant.getUri()); + } + @Test + public void constant_uri_only_supports_http() throws Exception { + RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + SearchBuilder searchBuilder = new SearchBuilder(rankProfileRegistry); + thrown.expect(ParseException.class); + thrown.expectMessage("Encountered \" <IDENTIFIER> \"ftp \"\" at line 5, column 10.\n" + + "Was expecting:\n" + + " <URI_PATH> ..."); + searchBuilder.importString(joinLines( + "search test {", + " document test { }", + " constant foo {", + " type: tensor(x{})", + " uri: ftp:somwhere.far.away/in/another-galaxy", + " }", + "}" + )); + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java index 61c376a7256..39bf2ee02fd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/AddFileInterface.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; public interface AddFileInterface { + FileReference addUri(String uri, String relativePath); + FileReference addUri(String uri, String relativePath, FileReference reference); FileReference addFile(String relativePath); FileReference addFile(String relativePath, FileReference reference); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java index 82535143c89..0e7aa4a4fd2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/ApplicationFileManager.java @@ -3,6 +3,12 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; public class ApplicationFileManager implements AddFileInterface { @@ -24,4 +30,41 @@ public class ApplicationFileManager implements AddFileInterface { return master.addFile(new File(applicationDir, relativePath)); } + @Override + public FileReference addUri(String uri, String relativePath) { + download(uri, relativePath); + return addFile(relativePath); + } + + @Override + public FileReference addUri(String uri, String relativePath, FileReference reference) { + download(uri, relativePath); + return addFile(relativePath, reference); + } + + void download(String uri, String relativePath) { + File file = new File(applicationDir, relativePath); + FileOutputStream fos = null; + ReadableByteChannel rbc = null; + try { + Files.createDirectories(file.toPath().getParent()); + URL website = new URL(uri); + rbc = Channels.newChannel(website.openStream()); + fos = new FileOutputStream(file.getAbsolutePath()); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + } catch (IOException e) { + throw new IllegalArgumentException("Failed creating directory " + file.getParent(), e); + } finally { + try { + if (fos != null) { + fos.close(); + } + if (rbc != null) { + rbc.close(); + } + } catch (IOException e) { + throw new IllegalArgumentException("Failed closing down after downloading " + uri + " to " + file.getAbsolutePath()); + } + } + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java index 8f2cb194bbd..1fe72e27461 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/CombinedLegacyRegistry.java @@ -21,6 +21,12 @@ public class CombinedLegacyRegistry implements FileRegistry { } @Override + public FileReference addUri(String uri) { + FileReference reference = future.addUri(uri); + return reference; + } + + @Override public String fileSourceHost() { return future.fileSourceHost(); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java index 1a76454fbed..b0a802d831d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDBRegistry.java @@ -4,7 +4,10 @@ package com.yahoo.vespa.config.server.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.net.HostName; +import com.yahoo.text.Utf8; +import net.jpountz.xxhash.XXHashFactory; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -34,6 +37,17 @@ public class FileDBRegistry implements FileRegistry { }); } + public synchronized FileReference addUri(String uri, FileReference reference) { + String relativePath = uriToRelativeFile(uri); + Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(uri)); + return cachedReference.orElseGet(() -> { + FileReference newRef = manager.addUri(uri, relativePath, reference); + entries.add(new Entry(uri, newRef)); + fileReferenceCache.put(uri, newRef); + return newRef; + }); + } + @Override public synchronized FileReference addFile(String relativePath) { Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(relativePath)); @@ -46,6 +60,18 @@ public class FileDBRegistry implements FileRegistry { } @Override + public synchronized FileReference addUri(String uri) { + String relativePath = uriToRelativeFile(uri); + Optional<FileReference> cachedReference = Optional.ofNullable(fileReferenceCache.get(uri)); + return cachedReference.orElseGet(() -> { + FileReference newRef = manager.addUri(uri, relativePath); + entries.add(new Entry(uri, newRef)); + fileReferenceCache.put(uri, newRef); + return newRef; + }); + } + + @Override public String fileSourceHost() { return HostName.getLocalhost(); } @@ -55,4 +81,16 @@ public class FileDBRegistry implements FileRegistry { return entries; } + private static String uriToRelativeFile(String uri) { + String relative = "uri/" + String.valueOf(XXHashFactory.nativeInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0)); + if (uri.endsWith(".json")) { + relative += ".json"; + } else if (uri.endsWith(".json.lz4")) { + relative += ".json.lz4"; + } else if (uri.endsWith(".lz4")) { + relative += ".lz4"; + } + return relative; + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java index 6c2da338ef0..117bf3e236b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDistributionProvider.java @@ -26,6 +26,17 @@ public class FileDistributionProvider { ManagerWrapper(FileDistributionManager manager) { this.manager = manager; } + + @Override + public FileReference addUri(String uri, String relativePath) { + throw new IllegalStateException("addUri is not possible with legacy filedistribution."); + } + + @Override + public FileReference addUri(String uri, String relativePath, FileReference reference) { + throw new IllegalStateException("addUri is not possible with legacy filedistribution."); + } + @Override public FileReference addFile(String relativePath) { return new FileReference(manager.addFile(relativePath)); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index ea8405f6b65..731e343532a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -104,6 +104,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { log.log(LogLevel.WARNING, "Unexpected error when building model ", e); throw new InternalServerException(applicationId + ": Error loading model", e); } else { + log.log(LogLevel.WARNING, "Input error when building model ", e); throw new IllegalArgumentException(applicationId + ": Error loading model", e); } } else { diff --git a/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala b/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala index 69443c73b3a..6507b4c72f0 100644 --- a/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala +++ b/standalone-container/src/main/scala/com/yahoo/container/standalone/LocalFileDb.scala @@ -58,6 +58,10 @@ class LocalFileDb(appPath: Path) extends FileAcquirer with FileRegistry { override def export(): util.List[Entry] = { new java.util.ArrayList(fileReferenceToFile.keys.map{ (ref: FileReference) => new Entry(fileReferenceToFile.get(ref).get.getPath, ref)}.asJavaCollection) } + + override def addUri(uri: String): FileReference = { + throw new RuntimeException("addUri(uri: String) is not implemented here."); + } } object LocalFileDb { |