diff options
19 files changed, 133 insertions, 154 deletions
diff --git a/README.md b/README.md index a684aae70dc..c7947e1d686 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,20 @@ [](https://vespa.ai) -The open big data serving engine - Store, search, organize and make machine-learned inferences -over big data at serving time. +Search, make inferences in, and organize vectors, tensors, text and structured data, at serving time and any scale. -This is the primary repository for Vespa where all development is happening. -New production releases from this repository's master branch are made each weekday from Monday through Thursday. +This repository contains all the code required to build and run all of Vespa yourself, +and where you can see all development as it is happening. +All the content in this repository is licensed under the Apache 2.0 license. + +A new release of Vespa is made from this repository's master branch every morning CET Monday through Thursday. +Build status: [](https://cd.screwdriver.cd/pipelines/6386) - Home page: [https://vespa.ai](https://vespa.ai) - Documentation: [https://docs.vespa.ai](https://docs.vespa.ai) - Continuous build: [https://factory.vespa.oath.cloud](https://factory.vespa.oath.cloud) - Run applications in the cloud for free: [https://cloud.vespa.ai](https://cloud.vespa.ai) -Vespa build status: [](https://cd.screwdriver.cd/pipelines/6386) - ## Table of contents - [Background](#background) @@ -37,12 +38,12 @@ hundreds of thousands of queries from Vespa per second. ## Install -Run your own Vespa instance: [https://docs.vespa.ai/en/getting-started.html](https://docs.vespa.ai/en/getting-started.html) -Or deploy your Vespa applications to the cloud service: [https://cloud.vespa.ai](https://cloud.vespa.ai) +Deploy your Vespa applications to the cloud service: [https://cloud.vespa.ai](https://cloud.vespa.ai), +or run your own Vespa instance: [https://docs.vespa.ai/en/getting-started.html](https://docs.vespa.ai/en/getting-started.html) ## Usage -- The application created in the getting started guide is fully functional and production ready, but you may want to [add more nodes](https://docs.vespa.ai/en/multinode-systems.html) for redundancy. +- The application created in the getting started guides linked above are fully functional and production ready, but you may want to [add more nodes](https://docs.vespa.ai/en/multinode-systems.html) for redundancy. - See [developing applications](https://docs.vespa.ai/en/developer-guide.html) on adding your own Java components to your Vespa application. - [Vespa APIs](https://docs.vespa.ai/en/api.html) is useful to understand how to interface with Vespa - Explore the [sample applications](https://github.com/vespa-engine/sample-apps/tree/master) diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 2a48c1b128e..0d27ca07f85 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -700,9 +700,9 @@ eslint-visitor-keys "^3.3.0" "@eslint-community/regexpp@^4.6.1": - version "4.8.1" - resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.8.1.tgz#8c4bb756cc2aa7eaf13cfa5e69c83afb3260c20c" - integrity sha512-PWiOzLIUAjN/w5K17PoF4n6sKBw0gqLHPhywmYHP4t1VFQQVYeb1yWsJwnMVEMl3tUHME7X/SJPZLmtG7XBDxQ== + version "4.9.1" + resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.9.1.tgz#449dfa81a57a1d755b09aa58d826c1262e4283b4" + integrity sha512-Y27x+MBLjXa+0JWDhykM3+JE+il3kHKAEqabfEWq3SDhZjLYb6/BHL/JKFnH3fe207JaXkyDo685Oc2Glt6ifA== "@eslint/eslintrc@^2.1.2": version "2.1.2" @@ -719,10 +719,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@8.50.0": - version "8.50.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.50.0.tgz#9e93b850f0f3fa35f5fa59adfd03adae8488e484" - integrity sha512-NCC3zz2+nvYd+Ckfh87rA47zfu2QsQpvc6k1yzTk+b9KzRj0wkGa8LSoGOXN6Zv4lRf/EIoZ80biDh9HOI+RNQ== +"@eslint/js@8.51.0": + version "8.51.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.51.0.tgz#6d419c240cfb2b66da37df230f7e7eef801c32fa" + integrity sha512-HxjQ8Qn+4SI3/AFv6sOrDB+g6PpUTDwSJiQqOrnneEk8L71161srI9gjzzZvYVbzHiVg/BvcH95+cK/zfIt4pg== "@floating-ui/core@^1.4.2": version "1.5.0" @@ -2460,14 +2460,14 @@ eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4 integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== eslint@^8: - version "8.50.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.50.0.tgz#2ae6015fee0240fcd3f83e1e25df0287f487d6b2" - integrity sha512-FOnOGSuFuFLv/Sa+FDVRZl4GGVAAFFi8LecRsI5a1tMO5HIE8nCm4ivAlzt4dT3ol/PaaGC0rJEEXQmHJBGoOg== + version "8.51.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.51.0.tgz#4a82dae60d209ac89a5cff1604fea978ba4950f3" + integrity sha512-2WuxRZBrlwnXi+/vFSJyjMqrNjtJqiasMzehF0shoLaW7DzS3/9Yvrmq5JiT66+pNjiX4UBnLDiKHcWAr/OInA== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.6.1" "@eslint/eslintrc" "^2.1.2" - "@eslint/js" "8.50.0" + "@eslint/js" "8.51.0" "@humanwhocodes/config-array" "^0.11.11" "@humanwhocodes/module-importer" "^1.0.1" "@nodelib/fs.walk" "^1.2.8" @@ -2752,15 +2752,15 @@ find-up@^5.0.0: path-exists "^4.0.0" flat-cache@^3.0.4: - version "3.1.0" - resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.1.0.tgz#0e54ab4a1a60fe87e2946b6b00657f1c99e1af3f" - integrity sha512-OHx4Qwrrt0E4jEIcI5/Xb+f+QmJYNj2rrK8wiIdQOIrB9WrrJL8cjZvXdXuBTkkEwEqLycb5BeZDV1o2i9bTew== + version "3.1.1" + resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.1.1.tgz#a02a15fdec25a8f844ff7cc658f03dd99eb4609b" + integrity sha512-/qM2b3LUIaIgviBQovTLvijfyOQXPtSRnRK26ksj2J7rzPIecePUIpJsZ4T02Qg+xiAEKIs5K8dsHEd+VaKa/Q== dependencies: - flatted "^3.2.7" + flatted "^3.2.9" keyv "^4.5.3" rimraf "^3.0.2" -flatted@^3.2.7: +flatted@^3.2.9: version "3.2.9" resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.9.tgz#7eb4c67ca1ba34232ca9d2d93e9886e611ad7daf" integrity sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ== @@ -2913,9 +2913,9 @@ globals@^11.1.0: integrity sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA== globals@^13.19.0: - version "13.22.0" - resolved "https://registry.yarnpkg.com/globals/-/globals-13.22.0.tgz#0c9fcb9c48a2494fbb5edbfee644285543eba9d8" - integrity sha512-H1Ddc/PbZHTDVJSnj8kWptIRSD6AM3pK+mKytuIVF4uoBV7rshFlhhvA58ceJ5wp3Er58w6zj7bykMpYXt3ETw== + version "13.23.0" + resolved "https://registry.yarnpkg.com/globals/-/globals-13.23.0.tgz#ef31673c926a0976e1f61dab4dca57e0c0a8af02" + integrity sha512-XAmF0RjlrjY23MA51q3HltdlGxUpXPvg0GioKiD9X6HD28iMjo2dKC8Vqwm7lne4GNr78+RHTfliktR6ZH09wA== dependencies: type-fest "^0.20.2" diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index e8f3d183abd..cff1d9727a5 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -98,7 +98,7 @@ <java-jjwt.vespa.version>0.11.5</java-jjwt.vespa.version> <java-jwt.vespa.version>4.4.0</java-jwt.vespa.version> <jaxb.runtime.vespa.version>4.0.3</jaxb.runtime.vespa.version> - <jetty.vespa.version>11.0.16</jetty.vespa.version> + <jetty.vespa.version>11.0.17</jetty.vespa.version> <jetty-servlet-api.vespa.version>5.0.2</jetty-servlet-api.vespa.version> <jimfs.vespa.version>1.3.0</jimfs.vespa.version> <jna.vespa.version>5.13.0</jna.vespa.version> @@ -111,10 +111,10 @@ <maven-archiver.vespa.version>3.6.1</maven-archiver.vespa.version> <maven-wagon.vespa.version>3.5.3</maven-wagon.vespa.version> <mimepull.vespa.version>1.10.0</mimepull.vespa.version> - <mockito.vespa.version>5.5.0</mockito.vespa.version> + <mockito.vespa.version>5.6.0</mockito.vespa.version> <mojo-executor.vespa.version>2.4.0</mojo-executor.vespa.version> <netty.vespa.version>4.1.99.Final</netty.vespa.version> - <netty-tcnative.vespa.version>2.0.61.Final</netty-tcnative.vespa.version> + <netty-tcnative.vespa.version>2.0.62.Final</netty-tcnative.vespa.version> <onnxruntime.vespa.version>1.15.1</onnxruntime.vespa.version> <opennlp.vespa.version>2.3.0</opennlp.vespa.version> <opentest4j.vespa.version>1.3.0</opentest4j.vespa.version> diff --git a/docproc/src/main/java/com/yahoo/docproc/Call.java b/docproc/src/main/java/com/yahoo/docproc/Call.java index 3840de63e13..40ca4845d63 100644 --- a/docproc/src/main/java/com/yahoo/docproc/Call.java +++ b/docproc/src/main/java/com/yahoo/docproc/Call.java @@ -126,17 +126,11 @@ public class Call implements Cloneable { private void unwrapSchemaMapping(Processing processing) { - final List<DocumentOperation> documentOperations = processing.getDocumentOperations(); - + List<DocumentOperation> documentOperations = processing.getDocumentOperations(); for (int i = 0; i < documentOperations.size(); i++) { - DocumentOperation documentOperation = documentOperations.get(i); - - if (documentOperation instanceof DocumentPut) { - DocumentPut putOperation = (DocumentPut) documentOperation; - - if (putOperation.getDocument() instanceof DocumentOperationWrapper) { - DocumentOperationWrapper proxy = (DocumentOperationWrapper) putOperation.getDocument(); - documentOperations.set(i, new DocumentPut(putOperation, ((DocumentPut)proxy.getWrappedDocumentOperation()).getDocument())); + if (documentOperations.get(i) instanceof DocumentPut putOperation) { + if (putOperation.getDocument() instanceof DocumentOperationWrapper proxy) { + documentOperations.set(i, new DocumentPut(putOperation, ((DocumentPut) proxy.getWrappedDocumentOperation()).getDocument())); } } } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java index caaff318cdd..e1e6164206f 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/MbusRequestContext.java @@ -32,6 +32,8 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Logger; +import static java.util.logging.Level.WARNING; + /** * @author Einar M R Rosenvinge */ @@ -68,7 +70,7 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { @Override public List<Processing> getProcessings() { if (deserialized.getAndSet(true)) { - return Collections.emptyList(); + return List.of(); } return processingFactory.fromMessage(requestMsg); } @@ -143,19 +145,17 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { @Override public String getServiceName() { String path = getUri().getPath(); - return path.substring(7, path.length()); + return path.substring(7); } @Override public boolean isProcessable() { - Message msg = requestMsg; - switch (msg.getType()) { - case DocumentProtocol.MESSAGE_PUTDOCUMENT: - case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: - case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: - return true; - } - return false; + return switch (requestMsg.getType()) { + case DocumentProtocol.MESSAGE_PUTDOCUMENT, + DocumentProtocol.MESSAGE_UPDATEDOCUMENT, + DocumentProtocol.MESSAGE_REMOVEDOCUMENT -> true; + default -> false; + }; } @Override @@ -180,14 +180,12 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { private void dispatchRequest(Message msg, String uriPath, ResponseHandler handler) { try { new RequestDispatch() { - @Override protected Request newRequest() { return new MbusRequest(request, uriCache.computeIfAbsent(uriPath, __ -> URI.create("mbus://remotehost" + uriPath)), msg); } - @Override public ContentChannel handleResponse(Response response) { return handler.handleResponse(response); @@ -195,7 +193,7 @@ public class MbusRequestContext implements RequestContext, ResponseHandler { }.dispatch(); } catch (Exception e) { dispatchResponse(Response.Status.INTERNAL_SERVER_ERROR); - e.printStackTrace(); + log.log(WARNING, "Failed to dispatch request", e); } } diff --git a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java index 0a7836ff6d3..aac22af1d11 100644 --- a/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java +++ b/docproc/src/main/java/com/yahoo/docproc/jdisc/messagebus/ProcessingFactory.java @@ -1,10 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.docproc.jdisc.messagebus; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Logger; import com.yahoo.component.ComponentId; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.concurrent.SystemTimer; @@ -22,6 +18,10 @@ import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.documentapi.messagebus.protocol.UpdateDocumentMessage; import com.yahoo.messagebus.Message; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Logger; + /** * @author Simon Thoresen Hult */ @@ -41,32 +41,28 @@ class ProcessingFactory { } public List<Processing> fromMessage(Message message) { - List<Processing> processings = new ArrayList<>(); - switch (message.getType()) { - case DocumentProtocol.MESSAGE_PUTDOCUMENT: { + return switch (message.getType()) { + case DocumentProtocol.MESSAGE_PUTDOCUMENT -> { PutDocumentMessage putMessage = (PutDocumentMessage) message; DocumentPut putOperation = new DocumentPut(createPutDocument(putMessage)); putOperation.setCondition(putMessage.getCondition()); putOperation.setCreateIfNonExistent(putMessage.getCreateIfNonExistent()); - processings.add(createProcessing(putOperation, message)); - break; + yield List.of(createProcessing(putOperation, message)); } - case DocumentProtocol.MESSAGE_UPDATEDOCUMENT: { + case DocumentProtocol.MESSAGE_UPDATEDOCUMENT -> { UpdateDocumentMessage updateMessage = (UpdateDocumentMessage) message; DocumentUpdate updateOperation = updateMessage.getDocumentUpdate(); updateOperation.setCondition(updateMessage.getCondition()); - processings.add(createProcessing(updateOperation, message)); - break; + yield List.of(createProcessing(updateOperation, message)); } - case DocumentProtocol.MESSAGE_REMOVEDOCUMENT: { + case DocumentProtocol.MESSAGE_REMOVEDOCUMENT -> { RemoveDocumentMessage removeMessage = (RemoveDocumentMessage) message; DocumentRemove removeOperation = new DocumentRemove(removeMessage.getDocumentId()); removeOperation.setCondition(removeMessage.getCondition()); - processings.add(createProcessing(removeOperation, message)); - break; + yield List.of(createProcessing(removeOperation, message)); } - } - return processings; + default -> List.of(); + }; } private Document createPutDocument(PutDocumentMessage msg) { diff --git a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java index 5ef97afec55..f72fb691852 100644 --- a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java +++ b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocument.java @@ -46,7 +46,7 @@ public class ProxyDocument extends Document implements DocumentOperationWrapper private final Map<String, String> fieldMap; private final Set<String> fieldsAllowed = new HashSet<>(); private final String docProcName; - private Document doc; + private final Document doc; public ProxyDocument(DocumentProcessor docProc, Document doc, Map<String, String> fieldMap) { super(doc); diff --git a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java index 642ae216687..56aaf7e3f56 100644 --- a/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java +++ b/docproc/src/main/java/com/yahoo/docproc/proxy/ProxyDocumentUpdate.java @@ -20,13 +20,13 @@ import java.util.Map; */ public class ProxyDocumentUpdate extends DocumentUpdate implements DocumentOperationWrapper { - private DocumentUpdate docU; + private final DocumentUpdate docU; /** * The field name map for schema mapping. The key is the field name that the docproc uses. * The value is the actual name of the field in the document. */ - private Map<String, String> fieldMap; + private final Map<String, String> fieldMap; public ProxyDocumentUpdate(DocumentUpdate docUpd, Map<String, String> fieldMap) { super(docUpd.getType(), docUpd.getId().toString()+"-schemamappedupdate"); diff --git a/document/src/main/java/com/yahoo/document/ExtendedField.java b/document/src/main/java/com/yahoo/document/ExtendedField.java index 7b897bd9845..7fc79527448 100644 --- a/document/src/main/java/com/yahoo/document/ExtendedField.java +++ b/document/src/main/java/com/yahoo/document/ExtendedField.java @@ -5,12 +5,12 @@ import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.StructuredFieldValue; /** - * This adds an Extractor to the Field that can be used to get access the backed value + * This adds an Extractor to the Field that can be used to access the backed value * used in the concrete document types. * @author baldersheim */ public class ExtendedField extends Field { - public static interface Extract { + public interface Extract { Object get(StructuredFieldValue doc); void set(StructuredFieldValue doc, Object value); } diff --git a/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java index 0dedfa72fc4..2f021632aef 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/LongFieldValue.java @@ -26,7 +26,7 @@ public final class LongFieldValue extends NumericFieldValue { private long value; public LongFieldValue() { - this(0l); + this(0L); } public LongFieldValue(long value) { @@ -50,7 +50,7 @@ public final class LongFieldValue extends NumericFieldValue { @Override public void clear() { - value = 0l; + value = 0L; } @Override diff --git a/document/src/main/java/com/yahoo/document/datatypes/MapFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/MapFieldValue.java index 01cd092bf57..18041ad1fab 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/MapFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/MapFieldValue.java @@ -22,7 +22,7 @@ import java.util.HashSet; /** - * Vespa map. Backed by and and parametrized by FieldValue + * Vespa map. Backed by and parametrized by FieldValue * * @author vegardh */ diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index e5f6453c581..d63ac7c194e 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -30,12 +30,15 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.List; import java.util.Map; +import java.util.Set; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -550,8 +553,8 @@ public class DocumentTestCase extends DocumentTestCaseBase { doc.iterateNested(path, 0, handler); FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray"); - assertTrue(((Array)fv).contains(new IntegerFieldValue(32))); - assertEquals(4, ((Array)fv).size()); + assertTrue(((Array<?>)fv).contains(new IntegerFieldValue(32))); + assertEquals(4, ((Array<?>)fv).size()); } { @@ -561,8 +564,8 @@ public class DocumentTestCase extends DocumentTestCaseBase { FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray"); - assertFalse(((Array)fv).contains(Integer.valueOf(12))); - assertEquals(3, ((Array)fv).size()); + assertFalse(((Array<?>)fv).contains(12)); + assertEquals(3, ((Array<?>)fv).size()); } { @@ -571,7 +574,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { doc.iterateNested(path, 0, handler); FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray"); - assertEquals(0, ((Array)fv).size()); + assertEquals(0, ((Array<?>)fv).size()); } { @@ -580,7 +583,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { doc.iterateNested(path, 0, handler); FieldValue fv = doc.getRecursiveValue("l1s1.structmap.value.smap"); - assertFalse(((MapFieldValue)fv).contains(new StringFieldValue("leonardo"))); + assertFalse(((MapFieldValue<?, ?>)fv).contains(new StringFieldValue("leonardo"))); } { @@ -589,7 +592,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { doc.iterateNested(path, 0, handler); FieldValue fv = doc.getRecursiveValue("l1s1.wset"); - assertFalse(((WeightedSet)fv).contains(new StringFieldValue("foo"))); + assertFalse(((WeightedSet<?>)fv).contains(new StringFieldValue("foo"))); } } @@ -648,7 +651,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { public void validateCppDoc(Document doc) { validateCppDocNotMap(doc); - MapFieldValue map = (MapFieldValue)doc.getFieldValue("mapfield"); + MapFieldValue<?, ?> map = (MapFieldValue<?, ?>)doc.getFieldValue("mapfield"); assertEquals(map.get(new StringFieldValue("foo1")), new StringFieldValue("bar1")); assertEquals(map.get(new StringFieldValue("foo2")), new StringFieldValue("bar2")); } @@ -734,7 +737,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { } @Test - public void testSerializeDeserialize() { + public void testSerializeDeserialize() throws IOException { setUpSertestDocType(); Document doc = getSertestDocument(); @@ -746,11 +749,8 @@ public class DocumentTestCase extends DocumentTestCaseBase { data.flip(); - try { - FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat"); + try (FileOutputStream fos = new FileOutputStream("src/test/files/testser.dat")) { fos.write(data.array(), 0, data.remaining()); - fos.close(); - } catch (Exception e) { } Document doc2 = docMan.createDocument(data); @@ -965,12 +965,11 @@ public class DocumentTestCase extends DocumentTestCaseBase { assertEquals(parsed.get("id"), "id:ns:sertest::foobar"); assertTrue(parsed.get("fields") instanceof Map); Object fieldMap = parsed.get("fields"); - if (fieldMap instanceof Map) { - Map<?, ?> fields = (Map<?, ?>) fieldMap; + if (fieldMap instanceof Map<?, ?> fields) { assertEquals(fields.get("mailid"), "emailfromalicetobob"); assertEquals(fields.get("date"), -2013512400); assertTrue(fields.get("docindoc") instanceof Map); - assertTrue(fields.keySet().containsAll(Arrays.asList("mailid", "date", "attachmentcount", "rawfield", "weightedfield", "docindoc", "mapfield", "myboolfield"))); + assertTrue(fields.keySet().containsAll(List.of("mailid", "date", "attachmentcount", "rawfield", "weightedfield", "docindoc", "mapfield", "myboolfield"))); } } @@ -1095,24 +1094,15 @@ public class DocumentTestCase extends DocumentTestCaseBase { @Test public void testRequireThatDocumentWithIdSchemaIdChecksType() { DocumentType docType = new DocumentType("mytype"); - try { - new Document(docType, "id:namespace:mytype::foo"); - } catch (Exception e) { - fail(); - } - - try { - new Document(docType, "id:namespace:wrong-type::foo"); - fail(); - } catch (IllegalArgumentException e) { - } + new Document(docType, "id:namespace:mytype::foo"); + assertThrows(IllegalArgumentException.class, + () -> new Document(docType, "id:namespace:wrong-type::foo")); } - private class MyDocumentReader implements DocumentReader { + private static class MyDocumentReader implements DocumentReader { @Override - public void read(Document document) { - } + public void read(Document document) { } @Override public DocumentId readDocumentId() { @@ -1129,22 +1119,17 @@ public class DocumentTestCase extends DocumentTestCaseBase { @Test public void testRequireThatChangingDocumentTypeChecksId() { MyDocumentReader reader = new MyDocumentReader(); + Document doc = new Document(reader); doc.setId(new DocumentId("id:namespace:mytype::foo")); DocumentType docType = new DocumentType("mytype"); - try { - doc.setDataType(docType); - } catch (Exception e) { - fail(); - } - doc = new Document(reader); - doc.setId(new DocumentId("id:namespace:mytype::foo")); + doc.setDataType(docType); + + Document notOkDoc = new Document(reader); + notOkDoc.setId(new DocumentId("id:namespace:mytype::foo")); DocumentType wrongType = new DocumentType("wrongtype"); - try { - doc.setDataType(wrongType); - fail(); - } catch (IllegalArgumentException e) { - } + assertThrows(IllegalArgumentException.class, + () -> notOkDoc.setDataType(wrongType)); } @Test diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 830b7f4ed33..d11adbe696a 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -297,12 +297,15 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private void refreshIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile, IdentityDocument doc, IdentityType identityType, AthenzIdentity identity) { - KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); - Pkcs10Csr csr = csrGenerator.generateInstanceCsr( - identity, doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); - try { + // Do not rotate private key on every refresh. + // TODO: rotate key pair only on Vespa upgrade or similar + PrivateKey privateKey = readPrivateKeyFromFile(privateKeyFile); + KeyPair keyPair = KeyUtils.toKeyPair(privateKey); + CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); + Pkcs10Csr csr = csrGenerator.generateInstanceCsr( + identity, doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); + // Allow all zts hosts while removing SIS HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; try (ZtsClient ztsClient = ztsClient(doc.ztsUrl(), privateKeyFile, certificateFile, ztsHostNameVerifier)) { @@ -347,6 +350,11 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { return X509CertificateUtils.fromPem(pemEncodedCertificate); } + private static PrivateKey readPrivateKeyFromFile(ContainerPath privateKeyFile) throws IOException { + String pemEncodedKey = new String(Files.readAllBytes(privateKeyFile)); + return KeyUtils.fromPemEncodedPrivateKey(pemEncodedKey); + } + private static boolean isCertificateExpired(Instant expiry, Instant now) { return now.isAfter(expiry.minus(EXPIRY_MARGIN)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 560abb1f0e8..c5d59e204e1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -618,17 +618,9 @@ public class Nodes { * @return the nodes in their new state */ public List<Node> restartActive(Predicate<Node> filter) { - return restart(NodeFilter.in(Set.of(Node.State.active)).and(filter)); - } - - /** - * Increases the restart generation of the any nodes matching given filter. - * - * @return the nodes in their new state - */ - public List<Node> restart(Predicate<Node> filter) { - return performOn(filter, (node, lock) -> write(node.withRestart(node.allocation().get().restartGeneration().withIncreasedWanted()), - lock)); + return performOn(NodeFilter.in(Set.of(State.active)).and(filter), + (node, lock) -> write(node.withRestart(node.allocation().get().restartGeneration().withIncreasedWanted()), + lock)); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 132cd0e6d67..1581577c622 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -15,7 +15,6 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; -import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; import com.yahoo.jdisc.Metric; import com.yahoo.transaction.Mutex; @@ -145,7 +144,10 @@ public class NodeRepositoryProvisioner implements Provisioner { @Override public void restart(ApplicationId application, HostFilter filter) { - nodeRepository.nodes().restartActive(ApplicationFilter.from(application).and(NodeHostFilter.from(filter))); + List<Node> updated = nodeRepository.nodes().restartActive(ApplicationFilter.from(application).and(NodeHostFilter.from(filter))); + if (updated.isEmpty()) { + throw new IllegalArgumentException("No matching nodes found"); + } } @Override diff --git a/parent/pom.xml b/parent/pom.xml index 244d6b07710..b1adebbaf3d 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -317,7 +317,7 @@ --> <groupId>org.openrewrite.maven</groupId> <artifactId>rewrite-maven-plugin</artifactId> - <version>5.8.0</version> + <version>5.8.1</version> <configuration> <activeRecipes> <recipe>org.openrewrite.java.testing.junit5.JUnit5BestPractices</recipe> diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java index e97409b40ef..fd297c291c2 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/client/AthenzIdentityProviderImpl.java @@ -278,17 +278,17 @@ public final class AthenzIdentityProviderImpl extends AbstractComponent implemen identity, role, athenzUniqueInstanceId, null, keyPair); try (ZtsClient client = createZtsClient()) { X509Certificate roleCertificate = client.getRoleCertificate(role, csr); - updateRoleKeyManager(role, roleCertificate); + updateRoleKeyManager(role, keyPair.getPrivate(), roleCertificate); log.info(String.format("Requester role certificate for role %s, expires: %s", role.toResourceNameString(), roleCertificate.getNotAfter().toInstant().toString())); return roleCertificate; } } - private void updateRoleKeyManager(AthenzRole role, X509Certificate certificate) { + private void updateRoleKeyManager(AthenzRole role, PrivateKey privateKey, X509Certificate certificate) { MutableX509KeyManager keyManager = roleKeyManagerCache.computeIfAbsent(role, r -> new MutableX509KeyManager()); keyManager.updateKeystore( KeyStoreBuilder.withType(PKCS12) - .withKeyEntry("default", autoReloadingX509KeyManager.getCurrentCertificateWithKey().privateKey(), certificate) + .withKeyEntry("default", privateKey, certificate) .build(), new char[0]); } diff --git a/vespajlib/src/main/java/com/yahoo/net/Url.java b/vespajlib/src/main/java/com/yahoo/net/Url.java index e6e7512c9f8..81959f58b6f 100644 --- a/vespajlib/src/main/java/com/yahoo/net/Url.java +++ b/vespajlib/src/main/java/com/yahoo/net/Url.java @@ -10,9 +10,9 @@ import java.util.regex.Pattern; public class Url { private static final Pattern pattern = Pattern.compile( - //12 3 456 7 8 9ab c d e f g h i j - // 2 1 6 87 5 c b ed a4 f hg ji - "^(([^:/?#]+):)?(//((([^:@/?#]+)(:([^@/?#]+))?@))?(((\\[([^\\]]+)\\]|[^:/?#]+)(:([^/?#]+))?)))?([^?#]+)?(\\?([^#]*))?(#(.*))?"); + //12 3 45 6 7 89 a b c d e f g h + // 2 1 5 76 4 a 9 cb 8 d fe hg + "^(([^:/?#]+):)?(//(([^:@/?#]+)(:([^@/?#]+))?@)?((\\[([^]]+)]|[^:/?#]+)(:([^/?#]+))?))?([^?#]+)?(\\?([^#]*))?(#(.*))?"); private final String image; private final int schemeBegin; private final int schemeEnd; @@ -122,17 +122,17 @@ public class Url { if (!matcher.matches()) { throw new IllegalArgumentException("Malformed URL."); } - String host = matcher.group(12); + String host = matcher.group(10); if (host == null) { - host = matcher.group(11); + host = matcher.group(9); } if (host == null) { - host = matcher.group(9); + host = matcher.group(8); } - String port = matcher.group(14); - return new Url(matcher.group(2), matcher.group(6), matcher.group(8), host, - port != null ? Integer.valueOf(port) : null, matcher.group(15), matcher.group(17), - matcher.group(19)); + String port = matcher.group(12); + return new Url(matcher.group(2), matcher.group(5), matcher.group(7), host, + port != null ? Integer.valueOf(port) : null, matcher.group(13), matcher.group(15), + matcher.group(17)); } public int getSchemeBegin() { diff --git a/zkfacade/src/main/java/com/yahoo/vespa/curator/SingletonManager.java b/zkfacade/src/main/java/com/yahoo/vespa/curator/SingletonManager.java index f349ee8e9a4..0c7956de067 100644 --- a/zkfacade/src/main/java/com/yahoo/vespa/curator/SingletonManager.java +++ b/zkfacade/src/main/java/com/yahoo/vespa/curator/SingletonManager.java @@ -38,6 +38,8 @@ import static java.util.logging.Level.WARNING; class SingletonManager { private static final Logger logger = Logger.getLogger(SingletonManager.class.getName()); + private static final String partPattern = "[a-zA-Z0-9$_]([a-zA-Z0-9$_-]+){0,63}"; + private static final Pattern idPattern = Pattern.compile(partPattern + "(\\." + partPattern + ")*"); private final Curator curator; private final Clock clock; @@ -55,7 +57,8 @@ class SingletonManager { } synchronized CompletableFuture<?> register(String singletonId, SingletonWorker singleton) { - Validation.requireMatch(singletonId, "Singleton ID", Pattern.compile("[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}")); + Validation.requireMatch(singletonId, "Singleton ID", idPattern); + Validation.requireLength(singletonId, "Singleton ID", 1, 255); String old = registrations.putIfAbsent(singleton, singletonId); if (old != null) throw new IllegalArgumentException(singleton + " already registered with ID " + old); count.merge(singletonId, 1, Integer::sum); |